JOLT spec on adding default values based on a condition - jolt

If my input contains "WorkflowCategory" in "metadata", then output should contain workflow.workflowInputProperties with specified default values - having duplicate values (like "" string, 3 etc). If not, workflow.workflowInputProperties should not be added.
Input 1
{
"template": false,
"active": true,
"metadata": [
{
"value": "bank_",
"key": "AssetNamePrefix"
},
{
"value": "-BERG",
"key": "SuffixForPublicId"
},
{
"value": "false",
"key": "CORSEnabled"
},
{
"value": "Capture",
"key": "WorkflowCategory"
},
{
"value": "HD",
"key": "Features"
}
],
"description": "Template for working with PRI",
"name": "prof_name",
"type": "Live",
"id": "BNK056003413",
"version": 6
}
Input 2
{
"template": false,
"active": true,
"metadata": [
{
"value": "HD",
"key": "Features"
}
],
"description": "Live Template",
"name": "Live_HD",
"type": "Live",
"id": "BNK007596994",
"version": 1
}
For Input 1, output should be
{
"id" : "BNK056003413",
"name" : "prof_name",
"metadataSet" : {
"description" : "Template for working with PRI",
"type" : "Live"
},
"workflow" : {
"workflowInputProperties" : {
"assetNamePrefix" : "bank_",
"recordId" : "",
"sourceUri":"",
"processingUri": "",
"recorderType": "ABC",
"completionTimeout": 600
"loopBackTimer": 10,
"numberOfRetries": 3,
"numberOfRetriesForScheduling": 3,
"scheduleDelay" : 3600
},
}
}
For Input 2, output should be as follows, without workflow.workflowInputProperties
{
"id" : "BNK007596994",
"name" : "Live_HD",
"metadataSet" : {
"description" : "Live Template",
"type" : "Live"
}
"features" : "HD"
}

You should add below code to the rest of your spec. The only issue is that I cannot put empty string and I change it into space. I'll try to figure it out.
[
{
"operation": "shift",
"spec": {
"metadata": {
"*": {
"key": {
"WorkflowCategory": {
"#bank_": "workflow.workflowInputProperties.assetNamePrefix",
"# ": [
"workflow.workflowInputProperties.recordId",
"workflow.workflowInputProperties.sourceUri",
"workflow.workflowInputProperties.processingUri"
],
"#ABC": "workflow.workflowInputProperties.recorderType",
"#600": "workflow.workflowInputProperties.completionTimeout",
"#10": "workflow.workflowInputProperties.loopBackTimer",
"#3": ["workflow.workflowInputProperties.numberOfRetries",
"workflow.workflowInputProperties.numberOfRetriesForScheduling"],
"#3600": "workflow.workflowInputProperties.scheduleDelay"
}
}
}
}
}
},
{
"operation": "modify-overwrite-beta",
"spec": {
"workflow": {
"workflowInputProperties": {
"completionTimeout": "=toInteger",
"loopBackTimer": "=toInteger",
"numberOfRetries": "=toInteger",
"numberOfRetriesForScheduling": "=toInteger",
"scheduleDelay": "=toInteger"
}
}
}
}
]

Related

KAFKA connector Apply Tansform.filter.Value

I have create a connector to azureEventhub , it works fine to pull the data into the topic ,
My use case is to filter the messages that I m pullling basing on the message type .
Example :
{
"messageType": "Transformed",
"timeStamp": 1652113146105,
"payload": {
"externalId": "24323",
"equipmentType": "TemperatureSensor",
"measureType": "Temperature",
"normalizedData": {
"equipmentData": [
{
"key": "ReadingValue",
"value": 23,
"valueType": "number",
"measurementUnit": "celsius",
"measurementDateTime": "2022-05-09T16:18:34.0000000Z"
}
]
},
"dataProviderName": "LineMetrics"
},
},
{
"messageType": "IntegratorGenericDataStream",
"timeStamp": 1652113146103,
"payload": {
"dataSource": {
"type": "sensor",
},
"dataPoints": [
{
"type": "Motion",
"value": 0,
"valueType": "number",
"dateTime": "2022-05-09T16:18:37.0000000Z",
"unit": "count"
}
],
"dataProvider": {
"id": "ba84cbdb-cbf8-4d4f-9a55-93b43f671b5a",
"name": "LineMetrics",
"displayName": "Line Metrics"
}
},
}
I wanted to apply a filter on a value like shown in the pic:
enter image description here
the error that appears to me :
enter image description here

JSON conversion using JOLT

I am trying to convert a JSON to different format using JOLT (using NiFi JoltTransformJson processor). For single JSON record, the JOLT am using is working fine in JOLT app demo whereas if i execute with multiple JSON records then I am not getting expected output in JOLT app demo. Could anyone correct me what additional changes I need to do in JOLT spec to handle multiple JSON records.
sample input json
[
{
"pool": {
"field": [
{
"name": "BillingDay",
"value": "12"
},
{
"name": "Custom1",
"value": "POOL_BASE_PLAN_3GB"
}
]
},
"usage": {
"version": "3",
"quota": {
"name": "POOL_TOP_UP_1GB_2",
"cid": "5764888998010953848"
}
},
"state": {
"version": "1",
"property": [
{
"name": "SMS_RO_TOP",
"value": "1"
},
{
"name": "BillingTimeStamp",
"value": "2020-06-12T01:00:05"
},
{
"name": "timereset",
"value": "2020-01-12T00:35:53"
}
]
}
},
{
"pool": {
"field": [
{
"name": "PoolID",
"value": "111100110000003505209"
},
{
"name": "BillingDay",
"value": "9"
}
]
},
"usage": {
"version": "3"
},
"state": {
"version": "1",
"property": [
{
"name": "BillingTimeStamp",
"value": "2020-06-09T01:00:05"
},
{
"name": "timereset",
"value": "2019-03-20T17:10:38"
}
]
}
}
]
JOLT using:
[
{
"operation": "modify-default-beta",
"spec": {
"state": {
"property": {
"name": "NOTAVAILABLE"
}
},
"usage": {
"quota": {
"name": "NOTAVAILABLE"
}
}
}
},
{
"operation": "shift",
"spec": {
"pool": {
"field": {
"*": {
"value": "pool_item.#(1,name)"
}
}
},
// remaining elements print as it is
"*": "&"
}
}
]
Expected output JSON:
[
{
"pool_item" : {
"BillingDay" : "12",
"Custom1" : "POOL_BASE_PLAN_3GB"
},
"usage" : {
"version" : "3",
"quota" : {
"name" : "POOL_TOP_UP_1GB_2",
"cid" : "5764888998010953848"
}
},
"state" : {
"version" : "1",
"property" : [ {
"name" : "SMS_RO_TOP",
"value" : "1"
}, {
"name" : "BillingTimeStamp",
"value" : "2020-06-12T01:00:05"
}, {
"name" : "timereset",
"value" : "2020-01-12T00:35:53"
} ]
}
},
{
"pool_item" : {
"BillingDay" : "9",
"PoolID" : "111100110000003505209"
},
"usage" : {
"version" : "3",
"quota" : {
"name" : "NOTAVAILABLE"
}
},
"state" : {
"version" : "1",
"property" : [ {
"name" : "SMS_RO_TOP",
"value" : "1"
}, {
"name" : "BillingTimeStamp",
"value" : "2020-06-12T01:00:05"
}, {
"name" : "timereset",
"value" : "2020-01-12T00:35:53"
} ]
}
}
]
This below jolt shift specification will work for your multiple json's in input array.
[
{
"operation": "shift",
"spec": {
"*": {
"pool": {
"field": {
"*": {
"value": "[&4].pool_item.#(1,name)"
}
}
},
"usage": "[&1].usage",
"state": "[&1].state"
}
}
}
]

how to validate datatype in jolt transformation

I'm new to jolt transformation. I was wondering if there is a way to do a validation on data type then proceed.
I'm processing a json to insert record into hbase. From source I'm getting timestamp repeated for the same resource id which I want to use for row key.
So I just retrieve the first timestamp and concate with resource id to create row key. But I have an issue when there is only one timestamp in the record i.e when its not a list. Appreciate if someone can help me how to handle this situation.
input data
{ "resource": {
"id": "200629068",
"name": "resource_name_1)",
"parent": {
"id": 200053744,
"name": "parent_name"
},
"properties": {
"AP_ifSpeed": "0",
"DisplaySpeed": "0 (NotApplicable)",
"description": "description"
}
},
"data": [
{
"metric": {
"id": "2215",
"name": "metric_name 1"
},
"timestamp": 1535064595000,
"value": 0
},
{
"metric": {
"id": "2216",
"name": "metric_name_2"
},
"timestamp": 1535064595000,
"value": 1
}
]
}
Jolt transformation
[{
"operation": "shift",
"spec": {
"resource": {
// "id": "resource_&",
"name": "resource_&",
"id": "resource_&",
"parent": {
"id": "parent_&",
"name": "parent_&"
},
"properties": {
"*": "&"
}
},
"data": {
"*": {
"metric": {
"id": {
"*": {
"#(3,value)": "&1"
}
},
"name": {
"*": {
"#(3,value)": "&1"
}
}
},
"timestamp": "timestamp"
}
}
}
}, {
"operation": "shift",
"spec": {
"timestamp": {
// get first element from list
"0": "&1"
},
"*": "&"
}
},
{
"operation": "modify-default-beta",
"spec": {
"rowkey": "=concat(#(1,resource_id),'_',#(1,timestamp))"
}
}
]
Output I'm getting
{ "resource_name" : "resource_name_1)",
"resource_id" : "200629068",
"parent_id" : 200053744,
"parent_name" : "parent_name",
"AP_ifSpeed" : "0",
"DisplaySpeed" : "0 (NotApplicable)",
"description" : "description",
"2215" : 0,
"metric_name 1" : 0,
"timestamp" : 1535064595000,
"2216" : 1,
"metric_name_2" : 1,
"rowkey" : "200629068_1535064595000"
}
when there is only one timestamp then I get
"rowkey" : "200629068_"
In your shift make the output "timestamp" always be an array, even if the incoming data array only has one element in it.
"timestamp": "timestamp[]"

Acumatica Rest API to get sales order with line item detail

I was doing a demo and could easily create and retrieve orders but was not able to get the expand parameter to work
For example
I created order SO003615 with:
URI http://localhost/Acumatica6/entity/Default/6.00.001/SalesOrder
{
"OrderType": { value: "SO" },
"CustomerID" : { value : "ACTIVESTAF" } ,
"LocationID" : { value : "MAIN" },
"Description" : { value : "Sample Order"},
"Details" :
[
{
"InventoryID" : {value: "AACOMPUT01"},
"Quantity" : {value: 2},
"UOM" : {value: "EA"},
"UnitPrice" : {value: 1000.99}
},
{
"InventoryID" : {value: "AALEGO500"},
"Quantity" : {value: 1}
}
]
}
Then I tried to get order with
URI http://localhost/Acumatica6/entity/Default/6.00.001/SalesOrder/SO/SO003615?expand=Details
But the line items are not in the result. What am I missing?
{
"id": "37c15980-f71d-4496-882d-6e05e4a50061",
"rowNumber": 1,
"note": "",
"BillingAddressOverride": {
"value": false
},
"BillingContactOverride": {
"value": false
},
"CreditHold": {
"value": false
},
"Currency": {
"value": "USD"
},
"CustomerID": {
"value": "ACTIVESTAF"
},
"CustomerOrder": {},
"Date": {
"value": "2017-03-16T00:00:00-04:00"
},
"Description": {
"value": "Sample Order 6"
},
"DestinationWarehouseID": {},
"ExternalReference": {},
"Hold": {
"value": false
},
"IsTaxValid": {
"value": false
},
"LastModified": {
"value": "2017-03-17T01:05:56.74-04:00"
},
"LocationID": {
"value": "MAIN"
},
"NewCard": {
"value": false
},
"OrderedQty": {
"value": 3
},
"OrderNbr": {
"value": "SO003615"
},
"OrderTotal": {
"value": 2101.98
},
"OrderType": {
"value": "SO"
},
"PaymentCardIdentifier": {},
"PaymentMethod": {
"value": "CHECK"
},
"PaymentRef": {},
"PreferredWarehouseID": {},
"Project": {
"value": "X"
},
"RequestedOn": {
"value": "2017-03-16T00:00:00-04:00"
},
"ShippingAddressOverride": {
"value": false
},
"ShippingContactOverride": {
"value": false
},
"ShipVia": {},
"Status": {
"value": "Open"
},
"TaxTotal": {
"value": 0
},
"custom": {},
"files": []
}
It is $expand, not expand. Try using the following
http://localhost/Acumatica6/entity/Default/6.00.001/SalesOrder/SO/003615?$expand=Details

mapping parent/child with _id mongo river elasticsearch

I've 3 collections in my database under mongo: shows - venues - dropdowns
shows are mapped like below
"show": {
"properties" : {
"description": {
"type": "string"
},
"image": {
"type": "string"
},
"site": {
"type": "string"
},
"title" : {
"type" : "multi_field",
"fields" : {
"title" : {"type" : "string", "index" : "analyzed"},
"raw_title" : {"type" : "string", "index" : "not_analyzed", "store": "no"}
}
}
}
}
venues like this
"venues": {
"properties" : {
"name" : {
"type" : "multi_field",
"fields" : {
"name" : {"type" : "string", "index" : "analyzed"},
"raw_name" : {"type" : "string", "index" : "not_analyzed", "store": "no"}
}
},
"city" : {
"type" : "multi_field",
"fields" : {
"city" : {"type" : "string", "index" : "analyzed"},
"raw_city" : {"type" : "string", "index" : "not_analyzed", "store": "no"}
}
},
"region" : {
"type" : "multi_field",
"fields" : {
"region" : {"type" : "string", "index" : "analyzed"},
"raw_region" : {"type" : "string", "index" : "not_analyzed", "store": "no"}
}
},
"state" : {
"type": "boolean"
}
}
}
and I've this model in mongo for dropdowns:
{
created: {
type: Date,
default: Date.now
},
analytics: {
type: String,
default: '',
trim: true
},
state: {
type: Boolean,
default: false,
index: true
},
show: {
type: Schema.ObjectId,
ref: 'Show'
},
venues:[{
venue:{
type: Schema.ObjectId,
ref: 'Venue',
index: true
},
site: {
type: String,
trim: true,
index: true
}
}]
}
I'd map dropdowns with parent/child schema into my index, but I can't understand if is possibile with ObjectId because I've tried with this mapping:
"dropdown": {
"properties" : {
"state": {
"type": "boolean"
},
"analytics": {
"type": "string"
},
"_parent":{
"type" : "show"
},
"venues" : {
"properties" : {
"venue" : {
"_parent": {
"type" : "venues"
}
}
},
"site" : {"type" : "string"}
}
}
}
But I received this error:
MapperParsingException[No type specified for property [show]]
There is anyway to setting up correctly my index?
Issue is that you're specifying _parent incorrectly. You have to set it not in properties field, but next to it. Please see documentation and example from it:
PUT /company
{
"mappings": {
"branch": {},
"employee": {
"_parent": {
"type": "branch"
}
}
}
}
So following that logic, I've taken your mappings, simplified it a bit and made it work:
PUT /test
{
"mappings": {
"show": {
"properties": {
"description": {
"type": "string"
},
"image": {
"type": "string"
},
"site": {
"type": "string"
},
"title": {
"type": "multi_field",
"fields": {
"title": {
"type": "string",
"index": "analyzed"
},
"raw_title": {
"type": "string",
"index": "not_analyzed",
"store": "no"
}
}
}
}
},
"venues": {
"properties": {
"name": {
"type": "multi_field",
"fields": {
"name": {
"type": "string",
"index": "analyzed"
},
"raw_name": {
"type": "string",
"index": "not_analyzed",
"store": "no"
}
}
},
"city": {
"type": "multi_field",
"fields": {
"city": {
"type": "string",
"index": "analyzed"
},
"raw_city": {
"type": "string",
"index": "not_analyzed",
"store": "no"
}
}
},
"region": {
"type": "multi_field",
"fields": {
"region": {
"type": "string",
"index": "analyzed"
},
"raw_region": {
"type": "string",
"index": "not_analyzed",
"store": "no"
}
}
},
"state": {
"type": "boolean"
}
}
},
"dropdown": {
"_parent": {
"type": "show"
},
"properties": {
"state": {
"type": "boolean"
},
"analytics": {
"type": "string"
},
"venues": {
"type": "object",
"_parent": {
"type": "venues"
},
"site": {
"type": "string"
}
}
}
}
}
}
I've tried this by myself on Elasticsearch 1.7.1 and it worked fine.
However, I'm not sure if you can declare _parent relationship inside nested documents as you did for venues. My mapping query didn't throw an error and accepted it. However, looking on how it got parsed in head plugin - _parent was eliminated and only object part remained as seen in screenshot:
If I tried to index it without specifying type - this error is thrown:
"MapperParsingException[mapping [dropdown]]; nested:
MapperParsingException[No type specified for property [venues]];