Jolt Transformation problems with null Elements - jolt

I have a problem with my Jolt transformation but no idea how to fix it.
I get "null" Element in the array I produce:
{
"Verkaufsprodukt": [
{
"Produkt": [
{
"Elementarprodukt": [
{
"ArtID": {
"bezeichnung": "b",
"value": "0302"
},
"VersichertePerson": {
"PartnerID": "1"
}
},
{
"ArtID": {
"bezeichnung": "f"
},
"VersichertePerson": {
"PartnerID": "1"
}
},
{
"ArtID": {
"bezeichnung": "c"
},
"VersichertePerson": {
"PartnerID": "1"
}
},
{
"ArtID": {
"bezeichnung": "a",
"value": "0301"
},
"VersichertePerson": {
"PartnerID": "1"
}
}
]
}
]
}
],
"Partner": [
{
"Name": "Holgerson",
"PartnerID": "1",
"Vorname": "Nils"
}
]
}
My result:
{
"vertragsdetails" : {
"versichertePersonen" : {
"versicherungssummenOderLeistungen" : [ null, {
"kennung" : "0302"
}, null, {
"kennung" : "0301"
} ]
}
}
}
Here is my spec:
[
{
"operation": "shift",
"spec": {
"Verkaufsprodukt": {
"*": {
"Produkt": {
"*": {
"Elementarprodukt": {
"*": {
"VersichertePerson": {
"PartnerID": {
"1": {
"#(3)": {
"ArtID": {
"value": "vertragsdetails.versichertePersonen.versicherungssummenOderLeistungen[&6].kennung"
}
}
}
}
}
}
}
}
}
}
}
}
}
]
I see, that the null elements comes from the "ArtID" elements without "vaules" but how can I get rid of them?
I tried a '"operation": "shift",' but that deleted also other elements I want to have.
Can somebody help? Thanks!

I found a soulotion. I added this to my spec:
{
"operation": "shift",
"spec": {
"vertragsdetails": {
"versichertePersonen": {
"versicherungssummenOderLeistungen": {
"*": {
"kennung": {
"#1": "vertragsdetails.versichertePersonen.versicherungssummenOderLeistungen[]"
}
}
}
"dataToKeep": "vertragsdetails.versichertePersonen.dataToKeep"
}
}
}
}

Related

JOLT Transformation with Arrays

Looking for help on Jolt transformation below
{
"consents": [
{
"consent-data": [
{
"consent-key": "DOB",
"consent-value": "02/2006"
}
],
"consent-id": "123",
"consent-indicator": "true",
"consent-type": "COPPA",
"resource-id": "456"
},
{
"consent-data": [
{
"consent-key": "DOB",
"consent-value": "02/2006"
}
],
"consent-id": "567",
"consent-indicator": "true",
"consent-type": "COPPA",
"resource-id": "678"
}
]
}
Expected Output :
{
"Consents": [
{
"consentDataResponse": [
{
"consentKey": "DOB",
"consentValue": "02/2006"
}
],
"consentId": "123",
"consentIndicator": "true",
"consentType": "COPPA",
"resourceId": "456"
},
{
"consentDataResponse": [
{
"consentKey": "DOB",
"consentValue": "02/2006"
}
],
"consentId": "567",
"consentIndicator": "true",
"consentType": "COPPA",
"resourceId": "678"
}
]
}
Trasformation iam using :
[
{
"operation": "shift",
"spec": {
"consents": {
"*": {
"consent-id": "consents[&1].retailerId",
"consent-indicator": "consents[&1].consentIndicator",
"consent-type": "consents[&1].consentType",
"resource-id": "consents[&1].resoureId",
"consent-data": {
// loop thru the orderItems array
"*": {
"consent-key": "consentDataResponse[&1].consentKey",
"consent-value": "consentDataResponse[&1].consentValue"
}
}
}
}
}
}
]
Current output :
{
"consentDataResponse" : [ {
"consentKey" : [ "DOB", "DOB1" ],
"consentValue" : [ "02/2006", "03/2006" ]
} ],
"consents" : [ {
"consentIndicator" : "true",
"consentType" : "COPPA",
"resoureId" : "22=",
"retailerId" : "11="
}, {
"consentIndicator" : "false",
"consentType" : "COPPA1",
"resoureId" : "44=",
"retailerId" : "33="
} ]
}
Note - Here, consents and consentDataResponse will be repeated, currently I am getting consentDataResponse as a separate object with consolidated values but it should be per consents. Could you please help me here
[
{
"operation": "shift",
"spec": {
"consents": {
"*": {
"consent-data": {
"*": {
"consent-key": "Consents[&3].consentDataResponse[&1].consentKey",
"consent-value": "Consents[&3].consentDataResponse[&1].consentValue"
}
},
"consent-id": "Consents[&1].consentId",
"consent-indicator": "Consents[&1].consentIndicator",
"consent-type": "Consents[&1].consentType",
"resource-id": "Consents[&1].resourceId"
}
}
}
}
]
You did almost correct and just added indexing inside the consent-data loop.

Array mapping with JOLT

First time JOLT user trying to restructure an array into a similar multi-level array. stackoverflow is asking for less code and more words, so I hope this extra sentence covers that.
Input JSON
{
"studentIdentifier": "453089029",
"studentRegistrationNumber": "753082022",
"parentIdentifier": "1001142760",
"parentFamilyName": "lastname1001142760",
"relationshipIdentifier": "1001142762",
"relationshipToStudent": "Mother"
},
{
"studentIdentifier": "453089193",
"studentRegistrationNumber": "753082123",
"parentIdentifier": "1001142760",
"parentFamilyName": "lastname1001142760",
"relationshipIdentifier": "1001159585",
"relationshipToStudent": "Mother"
}
]
Desired JSON output
{
"carer" : {
"relationships" : [ {
"relationshipIdentifier" : "1001142762",
"relationshipToStudent" : "Mother",
"student" : {
"studentIdentifier" : "453089029",
"studentRegistrationNumber" : "753082022"
}
}, {
"relationshipIdentifier" : "1001159585",
"relationshipToStudent" : "Mother",
"student" : {
"studentIdentifier" : "453089193",
"studentRegistrationNumber" : "753082123"
}
} ],
"parentIdentifier" : "1001142760",
"parentFamilyName" : "lastname1001142760"
}
}
JOLT spec so far
[
{
"operation": "shift",
"spec": {
"*": {
"#": "carer.relationships[].student",
"parent*": "carer.&"
}
}
},
{
"operation": "cardinality",
"spec": {
"*": {
"parent*": "ONE"
}
}
},
{
"operation": "remove",
"spec": {
"carer": {
"relationships": {
"*": {
"student": {
"parent*": "",
"relationship*": ""
}
}
}
}
}
}
]
Currently getting this result
{
"carer" : {
"relationships" : [ {
"student" : {
"studentIdentifier" : "453089029",
"studentRegistrationNumber" : "753082022"
}
}, {
"student" : {
"studentIdentifier" : "453089193",
"studentRegistrationNumber" : "753082123"
}
} ],
"parentIdentifier" : "1001142760",
"parentFamilyName" : "lastname1001142760"
}
}
How to map the missing relationship* fields at the right level or is there a better approach?
Spec:
[
{
"operation": "shift",
"spec": {
"*": {
"*": "&",
"student*": "carer.relationships[&1].student.&",
"relationship*": "carer.relationships[&1].&"
}
}
},
{
"operation": "cardinality",
"spec": {
"parent*": "ONE"
}
}
]

Jolt Transformation to match on multiple key values

I'm struggling to find the correct jolt spec to evaluate the values of two keys prior to creating an output. Any ideas?
Input:
{
"ticketInformation": {
"area": "001",
"site": "ABC",
"ticketType": "TC"
}
}
Spec:
[
{
"operation": "shift",
"spec": {
"ticketInformation": {
"area": {
"001": {
"#Works": "OneMatch"
}
},
"ticketType": {
"TC": {
"#OnlyEvaluatingTicketType": "HowToEvalueValuesOfTwoKeys"
}
}
}
}
}
]
Output:
{
"OneMatch" : "Works",
"HowToEvalueValuesOfTwoKeys" : "OnlyEvaluatingTicketType"
}
The desired output is the same as the actual output but ONLY if area=001 AND troubleType=TC.
I guess this is the spec you are looking for
[
{
"operation": "shift",
"spec": {
"ticketInformation": {
"area": {
"001": {
"#(2,ticketType)": {
"TC": {
"#Yes": "BothMatching"
}
}
}
}
}
}
}
]

REST API query string

I want to filter out the Sum_PKTS which value is lower than 10.
How could I merge the two query string?
Is it possible?
BTW, the "Sum_PKTS" field is sum by "field" : "Packet.
the goal is to filter the local IP and aggregate "packet" field, and finally filter the Sum_PKTS which value is lower than 10.
{
"range":{
"Sum_PKTS":{
"gte": 10
}
}
}
--
GET /_search
{
"size" : 0,
"query": {
"bool": {
"should": [
{
"match":{"IPV4_DST_ADDR":"192.168.0.0/16"}
},
{
"match":{"IPV4_SRC_ADDR":"192.168.0.0/16"}
}
],
"minimum_should_match": 1,
"must":[
{
"range":{
"#timestamp":{
"gte":"now-5m"
}
}
}
]
}
},
"aggs": {
"DST_Local_IP": {
"filter": {
"bool": {
"filter": {
"match":{"IPV4_DST_ADDR":"192.168.0.0/16"}
}
}
},
"aggs": {
"genres":{
"terms" : {
"field" : "IPV4_DST_ADDR" ,
"order" : { "Sum_PKTS" : "desc" }
},
"aggs":{
"Sum_PKTS": {
"sum" : { "field" : "Packet" }
}
}
}
}
},
"SRC_Local_IP": {
"filter": {
"bool": {
"filter": {
"match":{"IPV4_SRC_ADDR":"192.168.0.0/16"}
}
}
},
"aggs": {
"genres":{
"terms" : {
"field" : "IPV4_SRC_ADDR" ,
"order" : { "Sum_PKTS" : "desc" }
},
"aggs":{
"Sum_PKTS": {
"sum" : { "field" : "Packet" }
}
}
}
}
}
}
}
thank you in advance!
You can achieve what you want using a bucket selector pipeline aggregation (see the two Sum_PKTS_gte_10 aggregations below):
{
"size": 0,
"query": {
"bool": {
"should": [
{
"match": {
"IPV4_DST_ADDR": "192.168.0.0/16"
}
},
{
"match": {
"IPV4_SRC_ADDR": "192.168.0.0/16"
}
}
],
"minimum_should_match": 1,
"must": [
{
"range": {
"#timestamp": {
"gte": "now-5m"
}
}
}
]
}
},
"aggs": {
"DST_Local_IP": {
"filter": {
"bool": {
"filter": {
"match": {
"IPV4_DST_ADDR": "192.168.0.0/16"
}
}
}
},
"aggs": {
"genres": {
"terms": {
"field": "IPV4_DST_ADDR",
"order": {
"Sum_PKTS": "desc"
}
},
"aggs": {
"Sum_PKTS": {
"sum": {
"field": "Packet"
}
},
"Sum_PKTS_gte_10": {
"bucket_selector": {
"buckets_path": {
"sum_packets": "Sum_PKTS"
},
"script": "params.sum_packets >= 10"
}
}
}
}
}
},
"SRC_Local_IP": {
"filter": {
"bool": {
"filter": {
"match": {
"IPV4_SRC_ADDR": "192.168.0.0/16"
}
}
}
},
"aggs": {
"genres": {
"terms": {
"field": "IPV4_SRC_ADDR",
"order": {
"Sum_PKTS": "desc"
}
},
"aggs": {
"Sum_PKTS": {
"sum": {
"field": "Packet"
}
},
"Sum_PKTS_gte_10": {
"bucket_selector": {
"buckets_path": {
"sum_packets": "Sum_PKTS"
},
"script": "params.sum_packets >= 10"
}
}
}
}
}
}
}
}

Expected [START_OBJECT] under [filter]

I want to put double filter in aggs.
such like this.
"aggs": {
"download1" : {
"filter" : [
{ "term": { "IPV4_DST_ADDR":"192.168.0.159"}},
{ "range": { "LAST_SWITCHED": { "gte": "now-5m" } }}
],
"aggs" : {
"downlod_bytes" : { "sum" : { "field" : "IN_BYTES" } }
}
}
}
but it show me an error:
"error": {
"root_cause": [
{
"type": "parsing_exception",
"reason": "Expected [START_OBJECT] under [filter], but got a [START_ARRAY] in [download1]",
"line": 33,
"col": 24
}
]}
How can I do, thank you in advance!
You need to combine both queries with a bool/filter
{
"aggs": {
"download1": {
"filter": {
"bool": {
"filter": [
{
"term": {
"IPV4_DST_ADDR": "192.168.0.159"
}
},
{
"range": {
"LAST_SWITCHED": {
"gte": "now-5m"
}
}
}
]
}
},
"aggs": {
"downlod_bytes": {
"sum": {
"field": "IN_BYTES"
}
}
}
}
}
}