PostgreSQL - Parse Complex XML and load to tables

PostgreSQL - Parse Complex XML and load to tables - postgresql

Need a little bit of help here. I have a scenario where I want to parse an XML and insert data into a postgreSQL table. I need to parse around 100,000 XMLs an example of one of these XML is given below. However, the issue is with the unnest() function of XPATH, the query takes forever to load(20 mins approximately). I have tried XMLTABLE as well however the query requires multiple CTEs or temporary tables build upon XMLTABLE to handle nested nodes which is again slow. Is there a way to optimize? Query used is provided below -
SQL Query
INSERT INTO stateful_cc_ingest.job_details (ronumber, linlinecode, linlinecode_idx, hostitemid, lincause, lincause_idx, linstorytext, linstorytext_idx, linservicerequest, linservicerequest_idx, linaddonflag, linaddonflag_idx, store_id, closeddate) (
SELECT
(xpath('(//dns:RONumber/text())[1]'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))[1]::text::varchar AS ronumber,
unnest(xpath('(//dns:linLineCode/dns:V/text()[1])'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))::text AS linlinecode,
unnest(xpath('(//dns:linLineCode/dns:V/#Idx[1])'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))::text::integer AS linlinecode_idx,
(xpath('(//dns:HostItemID/text())[1]'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))[1]::text AS hostitemid,
unnest(xpath('(//dns:linCause/dns:V/text()[1])'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))::text AS lincause,
unnest(xpath('(//dns:linCause/dns:V/#Idx[1])'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))::text::integer AS lincause_idx,
unnest(xpath('(//dns:linStoryText/dns:V/text()[1])'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))::text AS linstorytext,
unnest(xpath('(//dns:linStoryText/dns:V/#Idx[1])'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))::text::integer AS linstorytext_idx,
unnest(xpath('(//dns:linServiceRequest/dns:V/text()[1])'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))::text AS linservicerequest,
unnest(xpath('(//dns:linServiceRequest/dns:V/#Idx[1])'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))::text::integer AS linservicerequest_idx,
unnest(xpath('(//dns:linAddOnFlag/dns:V/text()[1])'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))::text AS linaddonflag,
unnest(xpath('(//dns:linAddOnFlag/dns:V/#Idx[1])'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))::text::integer AS linaddonflag_idx,
data_detail.store_id,
(xpath('(//dns:ClosedDate/text())[1]'::text, data_detail.ro_data_xml, ARRAY[ARRAY['dns'::text, 'http://www.dmotorworks.com/service-repair-order-history'::text]]))[1]::text::date AS closeddate
FROM
stateful_cdk_source_raw.cdk_data_detail data_detail
CROSS JOIN fdw_stateful_service_configuration.configuration_value conf
WHERE
data_detail.store_id = in_store_id
AND closed_date IS NOT NULL
AND closed_date > conf.value::date
AND conf.short_key = 'CLD'
AND conf.party_id = in_store_id
AND data_detail.statuscode IS NOT NULL
AND data_detail.statusdesc IS NOT NULL)
ON CONFLICT (store_id,
ronumber,
closeddate,
linlinecode)
DO NOTHING;
END;
$function$
Sample XML
<service-repair-order-history
xmlns="http://www.dmotorworks.com/service-repair-order-history">
<prtQtyBackordered>
<V Idx="1">0</V>
<V Idx="2">0</V>
</prtQtyBackordered>
<prtQtyFilled>
<V Idx="1">1</V>
<V Idx="2">8</V>
</prtQtyFilled>
<prtQtyOnHand>
<V Idx="1">105</V>
<V Idx="2">-818</V>
</prtQtyOnHand>
<punMultivalueCount>1</punMultivalueCount>
<punTechNo>
<V Idx="1">1809</V>
</punTechNo>
<punWorkDate>
<V Idx="1">2021-01-22</V>
</punWorkDate>
<punTimeOn>
<V Idx="1">15:18:00</V>
</punTimeOn>
<punTimeOff>
<V Idx="1">16:06:00</V>
</punTimeOff>
<punDuration>
<V Idx="1">0.80</V>
</punDuration>
<punLineCode>
<V Idx="1">C B A</V>
</punLineCode>
<punWorkType>
<V Idx="1">W</V>
</punWorkType>
<BlockAutoMsg/>
<BookedTime>16:09:14</BookedTime>
<dedActualAmount/>
<dedLaborAmount/>
<dedLaborType/>
<dedLineCodes/>
<dedMaximumAmount/>
<dedMultivalueCount>0</dedMultivalueCount>
<dedPartsAmount/>
<dedSequenceNo/>
<disAppliedBy/>
<disClassOrType/>
<disDesc/>
<disDiscountID/>
<disLaborDiscount/>
<disLevel/>
<disLineCode/>
<disLopSeqNo/>
<disManagerOverride/>
<disMultivalueCount>0</disMultivalueCount>
<disOriginalDiscount/>
<disOverrideAmount/>
<disOverrideGPAmount/>
<disOverrideGPPercent/>
<disOverridePercent/>
<disOverrideTarget/>
<disPartsDiscount/>
<disSequenceNo/>
<disTotalDiscount/>
<EmailAddress>
<V mailto:idx="1">dnh#dlr.com</V>
</EmailAddress>
<EmailDesc>
<V Idx="1">HOME</V>
</EmailDesc>
<EmailMultivalueCount>1</EmailMultivalueCount>
<feeLOPorPartFlag/>
<feeLOPorPartSeqNo/>
<feeMcdPercentage/>
<feeMultivalueCount>0</feeMultivalueCount>
<feeSale/>
<feeSequenceNo/>
<feeType/>
<HasCustPayFlag>N</HasCustPayFlag>
<HasIntPayFlag>Y</HasIntPayFlag>
<HasWarrPayFlag>Y</HasWarrPayFlag>
<hrsFlagHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</hrsFlagHours>
<hrsMcdPercentage>
<V Idx="1">100</V>
<V Idx="2">100</V>
<V Idx="3">100</V>
</hrsMcdPercentage>
<hrsMultivalueCount>3</hrsMultivalueCount>
<hrsSequenceNo>
<V Idx="1">1</V>
<V Idx="2">2</V>
<V Idx="3">3</V>
</hrsSequenceNo>
<hrsTimeCardHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</hrsTimeCardHours>
<lbrActualHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.80</V>
</lbrActualHours>
<lbrComebackFlag>
<V Idx="1">N</V>
<V Idx="2">N</V>
<V Idx="3">N</V>
</lbrComebackFlag>
<lbrComebackRO/>
<lbrComebackSA/>
<lbrComebackTech/>
<lbrFlagHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</lbrFlagHours>
<lbrForcedShopCharge/>
<lbrLineCode>
<V Idx="1">A</V>
<V Idx="2">B</V>
<V Idx="3">C</V>
</lbrLineCode>
<lbrMultivalueCount>3</lbrMultivalueCount>
<lbrOperationType/>
<lbrOtherHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</lbrOtherHours>
<lbrSoldHours>
<V Idx="1">0.80</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</lbrSoldHours>
<LicenseNumber>NRP8569</LicenseNumber>
<linActualWork>
<V Idx="1">3</V>
<V Idx="2">3</V>
<V Idx="3">1</V>
</linActualWork>
<linAddOnFlag>
<V Idx="1">N</V>
<V Idx="2">N</V>
<V Idx="3">N</V>
</linAddOnFlag>
<linBookerNo>
<V Idx="1">1492</V>
<V Idx="2">1492</V>
<V Idx="3">1492</V>
</linBookerNo>
<linMultivalueCount>3</linMultivalueCount>
<linStorySequenceNo>
<V Idx="1">1</V>
</linStorySequenceNo>
<linStoryText>
<V Idx="1">8673 LOF AND TIRE ROTATION COMPLETE, LEARNED TIRE POSITIONS AND SET TO 42 PSI</V>
</linStoryText>
<LotLocation/>
<mlsCost/>
<mlsLaborType/>
<mlsLineCode/>
<mlsMcdPercentage/>
<mlsMultivalueCount>0</mlsMultivalueCount>
<mlsOpCode/>
<mlsOpCodeDesc/>
<mlsSale/>
<mlsSequenceNo/>
<ModelDesc>SILVERADO 1500</ModelDesc>
<OpenTime>15:13:51</OpenTime>
<payCPTotal/>
<payMultivalueCount>1</payMultivalueCount>
<payPaymentAmount>
<V Idx="1">0.00</V>
</payPaymentAmount>
<payPaymentCode>
<V Idx="1">CASH</V>
</payPaymentCode>
<payPaymentsMade/>
<PhoneDesc>
<V Idx="1">HomePhone</V>
<V Idx="2">MainPhone</V>
</PhoneDesc>
<PhoneExt/>
<PhoneMultivalueCount>2</PhoneMultivalueCount>
<PhoneNumber>
<V Idx="1">9726394152</V>
<V Idx="2">9726394152</V>
</PhoneNumber>
<prtExtendedCost>
<V Idx="1">3.56</V>
<V Idx="2">26.96</V>
</prtExtendedCost>
<prtExtendedSale>
<V Idx="1">0.04</V>
<V Idx="2">34.96</V>
</prtExtendedSale>
<prtLaborSequenceNo>
<V Idx="1">1</V>
<V Idx="2">1</V>
</prtLaborSequenceNo>
<prtLineCode>
<V Idx="1">A</V>
<V Idx="2">A</V>
</prtLineCode>
<prtMultivalueCount>2</prtMultivalueCount>
<prtSequenceNo>
<V Idx="1">1</V>
<V Idx="2">2</V>
</prtSequenceNo>
<punAlteredFlag>
<V Idx="1">N</V>
</punAlteredFlag>
<rapMultivalueCount>0</rapMultivalueCount>
<StatusCode>C98</StatusCode>
<StatusDesc>CLOSED</StatusDesc>
<totActualHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.80</V>
</totActualHours>
<totCoreCost>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totCoreCost>
<totCoreSale>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totCoreSale>
<totDiscount>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totDiscount>
<totFlagHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totFlagHours>
<totForcedShopCharge>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totForcedShopCharge>
<totLaborCount>
<V Idx="1">0</V>
<V Idx="2">1</V>
<V Idx="3">2</V>
</totLaborCount>
<totLaborDiscount>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totLaborDiscount>
<totLaborSalePostDed>
<V Idx="1">0.00</V>
<V Idx="2">15.00</V>
<V Idx="3">0.00</V>
</totLaborSalePostDed>
<totLubeCost>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totLubeCost>
<totLubeCount>
<V Idx="1">0</V>
<V Idx="2">0</V>
<V Idx="3">0</V>
</totLubeCount>
<totMiscCount>
<V Idx="1">0</V>
<V Idx="2">0</V>
<V Idx="3">0</V>
</totMiscCount>
<totMultivalueCount>3</totMultivalueCount>
<totOtherHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totOtherHours>
<totPartsCount>
<V Idx="1">0</V>
<V Idx="2">9</V>
<V Idx="3">0</V>
</totPartsCount>
<totPartsDiscount>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totPartsDiscount>
<totPartsSalePostDed>
<V Idx="1">0.00</V>
<V Idx="2">35.00</V>
<V Idx="3">0.00</V>
</totPartsSalePostDed>
<totPayType>
<V Idx="1">C</V>
<V Idx="2">W</V>
<V Idx="3">I</V>
</totPayType>
<totRoCost>
<V Idx="1">0.00</V>
<V Idx="2">47.32</V>
<V Idx="3">0.00</V>
</totRoCost>
<totRoSalePostDed>
<V Idx="1">0.00</V>
<V Idx="2">50.00</V>
<V Idx="3">0.00</V>
</totRoSalePostDed>
<totRoTax>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totRoTax>
<totShopChargeCost>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totShopChargeCost>
<totSubletCost>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totSubletCost>
<totSubletCount>
<V Idx="1">0</V>
<V Idx="2">0</V>
<V Idx="3">0</V>
</totSubletCount>
<totTimeCardHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totTimeCardHours>
<warAuthorizationCode/>
<warClaimType/>
<warConditionCode/>
<warFailedPartNo/>
<warFailedPartsCount/>
<warLaborSequenceNo/>
<warLineCode/>
<warMultivalueCount>0</warMultivalueCount>
<Zip>23661-2716</Zip>
<AddOnFlag>N</AddOnFlag>
<Address>107 HOLLYWOOD AVE</Address>
<ApptDate/>
<ApptFlag>N</ApptFlag>
<ApptTime/>
<BookedDate>2021-01-22</BookedDate>
<BookerNo>1492</BookerNo>
<Cashier>phillip</Cashier>
<CityStateZip>HAMPTON, VA 23661</CityStateZip>
<ClosedDate>2021-01-22</ClosedDate>
<ComebackFlag>N</ComebackFlag>
<Comments/>
<ContactEmailAddress/>
<ContactPhoneNumber/>
<CustNo>LG358915</CustNo>
<DeliveryDate>2020-07-08</DeliveryDate>
<ErrorLevel/>
<ErrorMessage/>
<EstComplDate>2021-01-22</EstComplDate>
<EstComplTime>16:24:00</EstComplTime>
<HostItemID>LG358915*851403</HostItemID>
<LastServiceDate/>
<Make>CHEV</Make>
<MakeDesc>CHEVROLET</MakeDesc>
<Mileage>8673</Mileage>
<MileageLastVisit>8673</MileageLastVisit>
<MileageOut>8673</MileageOut>
<Model>SILV15</Model>
<Name1>BECK,MATTHEW</Name1>
<Name2/>
<OpenDate>2021-01-22</OpenDate>
<OrigPromisedDate>2021-01-22</OrigPromisedDate>
<OrigPromisedTime/>
<OrigWaiterFlag>Y</OrigWaiterFlag>
<PostedDate>2021-01-22</PostedDate>
<PriorityValue>5686</PriorityValue>
<PromisedDate>2021-01-22</PromisedDate>
<PromisedTime>17:00:00</PromisedTime>
<PurchaseOrderNo/>
<RONumber>851403</RONumber>
<Remarks/>
<RentalFlag>N</RentalFlag>
<ServiceAdvisor>1492</ServiceAdvisor>
<SoldByDealerFlag>Y</SoldByDealerFlag>
<SpecialCustFlag>N</SpecialCustFlag>
<TagNo>TNPR8569</TagNo>
<VIN>3GCPYFED9LG358915</VIN>
<VehID>LG358915</VehID>
<VehicleColor/>
<VoidedDate/>
<WaiterFlag>Y</WaiterFlag>
<Year>2020</Year>
<disDebitAccountNo/>
<disDebitControlNo/>
<disDebitTargetCo/>
<disUserID/>
<feeCost/>
<feeFeeID/>
<feeLaborType/>
<feeLineCode/>
<feeOpCode/>
<feeOpCodeDesc/>
<hrsActualHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.80</V>
</hrsActualHours>
<hrsCost>
<V Idx="1">16.80</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</hrsCost>
<hrsHourType>
<V Idx="1">ST</V>
<V Idx="2">ST</V>
<V Idx="3">ST</V>
</hrsHourType>
<hrsLaborType>
<V Idx="1">WC06</V>
<V Idx="2">ICPA</V>
<V Idx="3">ICPA</V>
</hrsLaborType>
<hrsLineCode>
<V Idx="1">A</V>
<V Idx="2">B</V>
<V Idx="3">C</V>
</hrsLineCode>
<hrsOtherHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</hrsOtherHours>
<hrsPercentage>
<V Idx="1">100.00</V>
<V Idx="2">100.00</V>
<V Idx="3">100.00</V>
</hrsPercentage>
<hrsSale>
<V Idx="1">15.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</hrsSale>
<hrsSoldHours>
<V Idx="1">0.80</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</hrsSoldHours>
<hrsTechNo>
<V Idx="1">1809</V>
<V Idx="2">1809</V>
<V Idx="3">1809</V>
</hrsTechNo>
<lbrCost>
<V Idx="1">16.80</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</lbrCost>
<lbrLaborType>
<V Idx="1">WC06</V>
<V Idx="2">ICPA</V>
<V Idx="3">ICPA</V>
</lbrLaborType>
<lbrMcdPercentage>
<V Idx="1">100</V>
<V Idx="2">100</V>
<V Idx="3">100</V>
</lbrMcdPercentage>
<lbrOpCode>
<V Idx="1">0600723</V>
<V Idx="2">MPI</V>
<V Idx="3">CSI</V>
</lbrOpCode>
<lbrOpCodeDesc>
<V Idx="1">CHEVROLET 2YEAR SCHEDULED MAINTENANCE</V>
<V Idx="2">Multi Point Vehicle Inspection</V>
<V Idx="3">You may receive a Survey via email from Chevrolet in regard to your visit , we appreciate your feedback and thank you for choosing Hampton Chevrolet Thank you</V>
</lbrOpCodeDesc>
<lbrSale>
<V Idx="1">15.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</lbrSale>
<lbrSequenceNo>
<V Idx="1">1</V>
<V Idx="2">2</V>
<V Idx="3">3</V>
</lbrSequenceNo>
<lbrTechNo>
<V Idx="1">1809</V>
<V Idx="2">1809</V>
<V Idx="3">1809</V>
</lbrTechNo>
<lbrTimeCardHours>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</lbrTimeCardHours>
<linCampaignCode/>
<linCause>
<V Idx="1">FACTORY MAINT</V>
</linCause>
<linComebackFlag>
<V Idx="1">N</V>
<V Idx="2">N</V>
<V Idx="3">N</V>
</linComebackFlag>
<linComplaintCode>
<V Idx="1">0600723</V>
<V Idx="2">MPI</V>
<V Idx="3">CSI</V>
</linComplaintCode>
<linDispatchCode>
<V Idx="1">CS450</V>
<V Idx="2">CS450</V>
<V Idx="3">CS450</V>
</linDispatchCode>
<linEstDuration>
<V Idx="1">0.3</V>
<V Idx="2">0.3</V>
<V Idx="3">2.0</V>
</linEstDuration>
<linLineCode>
<V Idx="1">A</V>
<V Idx="2">B</V>
<V Idx="3">C</V>
</linLineCode>
<linServiceRequest>
<V Idx="1">CHEVROLET 2YEAR SCHEDULED MAINTENANCE</V>
<V Idx="2">Multi Point Vehicle Inspection</V>
<V Idx="3">You may receive a Survey via email from Chevrolet in regard to your visit , we appreciate your feedback and thank you for choosing Hampton Chevrolet Thank you</V>
</linServiceRequest>
<linStatusCode/>
<linStatusDesc/>
<linStoryEmployeeNo>
<V Idx="1">1809</V>
</linStoryEmployeeNo>
<mlsType/>
<payInsuranceFlag>
<V Idx="1">N</V>
</payInsuranceFlag>
<prtBin1>
<V Idx="1">A1</V>
<V Idx="2">TANK</V>
</prtBin1>
<prtClass>
<V Idx="1">1836</V>
<V Idx="2">8800</V>
</prtClass>
<prtComp>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
</prtComp>
<prtCompLineCode/>
<prtCoreCost>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
</prtCoreCost>
<prtCoreSale>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
</prtCoreSale>
<prtCost>
<V Idx="1">3.56</V>
<V Idx="2">3.37</V>
</prtCost>
<prtDesc>
<V Idx="1">(S)FILTER</V>
<V Idx="2">OIL</V>
</prtDesc>
<prtEmployeeNo>
<V Idx="1">170061</V>
<V Idx="2">170061</V>
</prtEmployeeNo>
<prtLaborType>
<V Idx="1">WC06</V>
<V Idx="2">WC06</V>
</prtLaborType>
<prtList>
<V Idx="1">7.12</V>
<V Idx="2">6.84</V>
</prtList>
<prtMcdPercentage>
<V Idx="1">100</V>
<V Idx="2">100</V>
</prtMcdPercentage>
<prtOutsideSalesmanNo/>
<prtPartNo>
<V Idx="1">19419227</V>
<V Idx="2">88865700</V>
</prtPartNo>
<prtQtyOrdered>
<V Idx="1">1</V>
<V Idx="2">8</V>
</prtQtyOrdered>
<prtQtySold>
<V Idx="1">1</V>
<V Idx="2">8</V>
</prtQtySold>
<prtSale>
<V Idx="1">0.04</V>
<V Idx="2">4.37</V>
</prtSale>
<prtSource>
<V Idx="1">191</V>
<V Idx="2">184</V>
</prtSource>
<prtSpecialStatus/>
<prtUnitServiceCharge>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
</prtUnitServiceCharge>
<rapApptID/>
<totLaborCost>
<V Idx="1">0.00</V>
<V Idx="2">16.80</V>
<V Idx="3">0.00</V>
</totLaborCost>
<totLaborSale>
<V Idx="1">0.00</V>
<V Idx="2">15.00</V>
<V Idx="3">0.00</V>
</totLaborSale>
<totLocalTax>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totLocalTax>
<totLubeSale>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totLubeSale>
<totMiscCost>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totMiscCost>
<totMiscSale>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totMiscSale>
<totPartsCost>
<V Idx="1">0.00</V>
<V Idx="2">30.52</V>
<V Idx="3">0.00</V>
</totPartsCost>
<totPartsSale>
<V Idx="1">0.00</V>
<V Idx="2">35.00</V>
<V Idx="3">0.00</V>
</totPartsSale>
<totShopChargeSale>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totShopChargeSale>
<totSoldHours>
<V Idx="1">0.00</V>
<V Idx="2">0.80</V>
<V Idx="3">0.00</V>
</totSoldHours>
<totStateTax>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totStateTax>
<totSubletSale>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totSubletSale>
<totSupp2Tax>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totSupp2Tax>
<totSupp3Tax>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totSupp3Tax>
<totSupp4Tax>
<V Idx="1">0.00</V>
<V Idx="2">0.00</V>
<V Idx="3">0.00</V>
</totSupp4Tax>
<warFailureCode/>
</service-repair-order-history>
Expected Output

Related

I have an error while working with logback-classic in scala

Here is the build.sbt file where i added the logback dependency
ThisBuild / version := "0.1.0-SNAPSHOT"
ThisBuild / scalaVersion := "2.13.8"
libraryDependencies ++= Seq(
"com.typesafe.akka" %% "akka-actor" % "2.6.3",
"ch.qos.logback" % "logback-classic" % "1.4.4" % Test
)
lazy val root = (project in file("."))
.settings(
name := "logBack"
)
Here is the logback.xml file, logBack is my project name
<configuration>
<appender name="logBack" class="ch.qos.logback.core.FileAppender">
<File>/tmp/logBack.log</File>
<encoder>
<pattern>%d{HH:mm:ss.SSS} %r [%thread] %logger{0} %level - %msg%n</pattern>
</encoder>
</appender>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} %r [%thread] %logger{0} %level - %msg%n</pattern>
</encoder>
</appender>
<logger name="interpretlogger" level="debug" additivity="false">
<appender-ref ref="STDOUT"/>
<appender-ref ref="Interpret"/>
</logger>
<root level="debug">
<appender-ref ref="logBack"/>
<appender-ref ref="STDOUT"/>
</root>
</configuration>
here is my scala file (logPractice)
import com.sun.org.slf4j.internal.LoggerFactory
object logPractice extends App{
val logger = LoggerFactory.getLogger(this.getClass)
logger.debug("Asif")
logger.error("Invalid Query")
}
Error
Exception in thread "main" java.lang.IllegalAccessError: class logPractice$ (in unnamed module #0x31dc339b) cannot access class com.sun.org.slf4j.internal.LoggerFactory (in module java.xml.crypto) because module java.xml.crypto does not export com.sun.org.slf4j.internal to unnamed module #0x31dc339b
at logPractice$.delayedEndpoint$logPractice$1(logPractice.scala:5)
at logPractice$delayedInit$body.apply(logPractice.scala:3)
at scala.Function0.apply$mcV$sp(Function0.scala:39)
at scala.Function0.apply$mcV$sp$(Function0.scala:39)
at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:17)
at scala.App.$anonfun$main$1(App.scala:76)
at scala.App.$anonfun$main$1$adapted(App.scala:76)
at scala.collection.IterableOnceOps.foreach(IterableOnce.scala:563)
at scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:561)
at scala.collection.AbstractIterable.foreach(Iterable.scala:926)
at scala.App.main(App.scala:76)
at scala.App.main$(App.scala:74)
at logPractice$.main(logPractice.scala:3)
at logPractice.main(logPractice.scala)
Moreover the object of logger don not run the logger.info() and generate and error

it has been resolved by importing the library
import org.slf4j.LoggerFactory

Eclipse Xquery For Complex payload

I have the input request like below
<Input>
<BusinessObjects>
<BusinessObject>
<BusinessIdentifiers>
<BusinessIdentifier>
<BKey>BuCode</BKey>
<BValue>CDC</BValue>
</BusinessIdentifier>
<BusinessIdentifier>
<BKey>BuType</BKey>
<BValue>123</BValue>
</BusinessIdentifier>
<BusinessIdentifier>
<BKey>CsmNo</BKey>
<BValue>857895</BValue>
</BusinessIdentifier>
</BusinessIdentifiers>
</BusinessObject>
<BusinessObject>
<BusinessIdentifiers>
<BusinessIdentifier>
<BKey>BuCode</BKey>
<BValue>CDC</BValue>
</BusinessIdentifier>
<BusinessIdentifier>
<BKey>BuType</BKey>
<BValue>123</BValue>
</BusinessIdentifier>
<BusinessIdentifier>
<BKey>CsmNo</BKey>
<BValue>34567</BValue>
</BusinessIdentifier>
</BusinessIdentifiers>
</BusinessObject>
</BusinessObjects>
</Input>
i need to form an output like below schema
<Output>
<BusinessObject>
<BIKey></BIKey>
<BKey></BIKey>
<Bvalue></Bvalue>
<BOID></BOID>
</BusinessObject>
</Output>
For the above payload
The output should be
<Output>
<BusinessObjects>
<BusinessObject>
<BIKey>CDC:123:857895|CDC:123:34567</BIKey>
<BKey>BUCode</BKey>
<Bvalue>CDC</Bvalue>
<BOID>CDC:123:857895</BOID>
</BusinessObject>
<BusinessObject>
<BIKey>CDC:123:857895|CDC:123:34567</BIKey>
<BKey>BUtype</BKey>
<Bvalue>123</Bvalue>
<BOID>CDC:123:857895</BOID>
</BusinessObject>
<BusinessObject>
<BIKey>CDC:123:857895|CDC:123:34567</BIKey>
<BKey>CSMNo</BKey>
<Bvalue>857895</Bvalue>
<BOID>CDC:123:857895</BOID>
</BusinessObject>
<BusinessObject>
<BIKey>CDC:123:857895|CDC:123:34567</BIKey>
<BKey>BUCode</BKey>
<Bvalue>CDC</Bvalue>
<BOID>CDC:123:34567</BOID>
</BusinessObject>
<BusinessObject>
<BIKey>CDC:123:857895|CDC:123:34567</BIKey>
<BKey>BUtype</BKey>
<Bvalue>123</Bvalue>
<BOID>CDC:123:34567</BOID>
</BusinessObject>
<BusinessObject>
<BIKey>CDC:123:857895|CDC:123:34567</BIKey>
<BKey>CSMNo</BKey>
<Bvalue>857895</Bvalue>
<BOID>CDC:123:34567</BOID>
</BusinessObject>
</BusinessObjects>
</Output>
i have tried below Xquery to get the same but ended up with errors or not meeting the requiremnt
<Ouput>
<BusinessObjects>
{
for $bi in Input/BusinessObjects/BusinessObject/BusinessIdentifiers/BusinessIdentifier
return
<BIKey>
{
string-join(
for $bo in Input/BusinessObjects/BusinessObject return string-join($bo/BusinessIdentifiers/BusinessIdentifier/BValue, '|'),
':'
)
}
</BIKey>
<BKey>data {$bi/Bkey}</BKey>
<Bvalue>data {$bi/Bvalue}</Bvalue>
for $bo in Input/BusinessObjects/BusinessObject return <BOID>{string-join($bo//BValue, ':')}<BOID>
}
</BusinessObjects>
</Ouput>
the description for the output fields as follows
BIKey-->it has formed with all the Bvalues of 'Business Identifier' concatenated with ':' and then for each businessobject it is separed with '|'
Bkey-->Straight mapping with bkey
Bvalue-->Straight mapping with Bvalue
BOID--> it has to formed for each businessobject, need to concatenate the values Bvalues of Business Identifiers with ':'
Any suggestions, i believe that i have to two complex loops in here, but not able to crack it.
Thanks for #Martin as he helped me to crack this using 'ancestor' axis, but some how eclispe is not identifying 'ancestor'.
Thanks

The BIKey will be the same for all of the output, so we can compute it outside of any loop and be done with it. We do so by joining all the BValues of a BusinessObject with :, and joining the result of this for each BusinessObject with | :
let $BIKey := string-join(for $bo in /Input/BusinessObjects/BusinessObject return string-join($bo/BusinessIdentifiers/BusinessIdentifier/BValue, ':'), '|')
The BOID will depend on the BusinessObject we're handling, so we need to loop over those and we can extract the BOID at that point :
let $BIKEY := [...]
for $bo in /Input/BusinessObjects/BusinessObject
let $BOID := string-join($bo/BusinessIdentifiers/BusinessIdentifier/BValue, ':')
The rest of the output fields depend on the BusinessIdentifier, so let's loop on these too and return a BusinessObject per BusinessIdentifier :
let $BIKEY := [...]
for $bo in [...]
let $BOID := [...]
return for $bi in $bo/BusinessIdentifiers/BusinessIdentifier/
return <BusinessObject>
<BIKey>{$BIKEY}</BIKey>
<BKey>{$bi/BKey}</BKey>
<Bvalue>{$bi/Value}</Bvalue>
<BOID>{$BOID}</BOID>
</BusinessObject>
Now all we need is to wrap those output BusinessObjects into their parent tags :
<output><BusinessObjects>{
let $BIKEY := string-join(for $bo in /Input/BusinessObjects/BusinessObject return string-join($bo/BusinessIdentifiers/BusinessIdentifier/BValue, ':'), '|')
for $bo in /Input/BusinessObjects/BusinessObject
let $BOID := string-join($bo/BusinessIdentifiers/BusinessIdentifier/BValue, ':')
return
for $bi in $bo/BusinessIdentifiers/BusinessIdentifier
return <BusinessObject>
<BIKey>{$BIKEY}</BIKey>
<BKey>{$bi/BKey/text()}</BKey>
<Bvalue>{$bi/BValue/text()}</Bvalue>
<BOID>{$BOID}</BOID>
</BusinessObject>
}</BusinessObjects></output>
You can test it here.

cannot log to console using scala-logging with logback.xml configuration on amazon-emr

I am using scala-logging with a logback.xml configuration file to send log messages to the console, but they do not appear. My code is running on an Amazon-EMR cluster and my code is called using spark-sumbit.
My build.sbt file contains the dependencies:
libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.1.7"
libraryDependencies += "com.typesafe.scala-logging" %% "scala-logging" % "3.5.0"
My src/main/resources/logback.xml contains:
<configuration>
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<root level="debug">
<appender-ref ref="console"/>
</root>
</configuration>
I am logging from a method in a class that resembles:
import com.typesafe.scalalogging.LazyLogging
class MyClass() extends LazyLogging {
def myMethod() = {
logger.debug("debug logging test")
logger.info("info logging test")
logger.warn("warn logging test")
logger.error("error logging test")
println("This message appears in console")
}
}
None of the messages at any log level appear. Note that when I use println method, the messages appear in the console/stdout.

Apache Thrift server logs

I have a server which has implemented in scala.
import org.apache.thrift.server.TServer
import org.apache.thrift.server.TThreadPoolServer
import org.apache.thrift.transport.TServerSocket
import org.apache.thrift.transport.TTransportException
import hms.config.demo.AppStoreConfig
object Server {
def main(args: Array[String]) {
start
}
private def start {
try {
val serverTransport: TServerSocket = new TServerSocket(7911)
val processor = new AppStoreConfig.Processor(new AppStoreConfigImpl)
val server: TServer = new TThreadPoolServer(new TThreadPoolServer.Args(serverTransport).processor(processor))
println("Starting server on port 7911 ...")
server.serve
}
catch {
case e: TTransportException => {
e.printStackTrace
}
}
}
}
I need to get logs from the server. That means how it handles the requests and respond to clients like tomcat server. Is there a way to achieve my task in thrift server?

It depends on slf4j as you can see here in "depends on". Here is documentation on how to configure slf4j: http://slf4j.org/faq.html. Basically you start by making a config file src/main/resources/logback.xml and putting something like this into it:
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<appender name="FILE" class="ch.qos.logback.core.FileAppender">
<file>/var/log/mysuperapp/supername.log</file>
<append>true</append>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<logger name="org.apache.thrift.server" level="info" />
<root level="info">
<appender-ref ref="FILE" />
</root>
</configuration>
Pick the appropriate level for desired package, add multiple lines for multiple packages. This requires you to have logback dependecy/jar present on classpath.

howto use the logback discriminator value to filter mongodb inserts

I have multiple threads generating log entries and I use the logback SiftingAppender to know who did what. Everything works fine and now im trying to save the log to mongodb.
In the mongodb the log needs to be saved into an embedded document array. Every user document have one embedded document that has an array of embedded documents containing log lines
Since im just started to learn logback this has to be some trial and error now.
In the below test logback.xml i have the file, consol and a custom appender.
My idea was that i could catch the SiftingAppender discriminator value in the custom appender append() method. Then the getMDCPropertyMap(); in the ILoggingEventgives me the MDC values but the question is if this is an efficient technic for what i want to do
I cannot see that logback has any native MDC-mongodb-appender or maybe i have missed something.
<configuration>
<appender name="SIFT-FILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<!-- in the absence of the class attribute, it is assumed that the
desired discriminator type is
ch.qos.logback.classic.sift.MDCBasedDiscriminator -->
<discriminator>
<key>userid</key>
<defaultValue>unknown</defaultValue>
</discriminator>
<sift>
<appender name="FILE-${userid}" class="ch.qos.logback.core.FileAppender">
<file>${userid}.log</file>
<append>true</append>
<layout class="ch.qos.logback.classic.PatternLayout">
<pattern>%d [%thread] %level %mdc %logger{35} - %msg%n</pattern>
</layout>
</appender>
</sift>
</appender>
<appender name="SIFT-STDOUT" class="ch.qos.logback.classic.sift.SiftingAppender">
<!-- in the absence of the class attribute, it is assumed that the
desired discriminator type is
ch.qos.logback.classic.sift.MDCBasedDiscriminator -->
<discriminator>
<key>userid</key>
<defaultValue>unknown</defaultValue>
</discriminator>
<sift>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%-4relative [%thread] %-5level %logger{35} ${userid}- %msg %n</pattern>
</encoder>
</appender>
</sift>
</appender>
<appender name="SIFT-MONGO" class="ch.qos.logback.classic.sift.SiftingAppender">
<!-- in the absence of the class attribute, it is assumed that the
desired discriminator type is
ch.qos.logback.classic.sift.MDCBasedDiscriminator -->
<discriminator>
<key>userid</key>
<defaultValue>unknown</defaultValue>
</discriminator>
<sift>
<appender name="MONGO" class="com.carlsberg.MongoAppender">
</appender>
</sift>
</appender>
<root level="DEBUG">
<appender-ref ref="SIFT-FILE" />
<appender-ref ref="SIFT-STDOUT" />
<appender-ref ref="SIFT-MONGO" />
</root>
</configuration>

I used the SiftingAppender discriminator value in the my custom appender like this.
public void append(ILoggingEvent event) {
Map<String, String> m = event.getMDCPropertyMap();
String id = (String) m.get("userid");
if(id != null){
Query<UserLog> query1 = mongo.createQuery(UserLog.class);
query1.field("lowerCaseUserName").equal(id.toLowerCase());
UpdateOperations<UserLog> up2 = mongo.createUpdateOperations
(UserLog.class).add("log", event.getLevel().levelStr +" "+ ft.format(event.getTimeStamp()) +" "+ event.getFormattedMessage(), true);
UpdateResults<UserLog> udr1 = mongo.update(query1, up2);
if(udr1.getError() != null){
System.out.print("ERROR CANNOT SAVE to UserLog ip adress for " + id);
}
}
}

We Keep Coding

iphone swift flutter scala powershell matlab mongodb postgresql perl eclipse

PostgreSQL - Parse Complex XML and load to tables - postgresql

Related

I have an error while working with logback-classic in scala

Eclipse Xquery For Complex payload

cannot log to console using scala-logging with logback.xml configuration on amazon-emr

Apache Thrift server logs

howto use the logback discriminator value to filter mongodb inserts

Categories

Resources