Need help with a JOLT specification for converting the JSON from the shared input format to the expected format.
[
{
"Header": {
"TeamCD": 2
},
"Player": [
{
"PlayerNumber": 1,
"ShortName": "John",
"UniformNO": 11,
"SmartStart": 20200201,
"SmartEnd": 99999999
},
{
"PlayerNumber": 2,
"ShortName": "James",
"UniformNO": 12,
"SmartStart": 20200201,
"SmartEnd": 99999999
}
]
}
]
Desired output:
[
{
"Field": "TeamCD",
"Type": "20"
"Value": 2,
"Vendor": "Vendor1"
},
{
"Field": "PlayerNumber",
"Type": "23"
"Value": 1,
"Vendor": "Vendor1"
},
{
"Field": "ShortName",
"Type": "24"
"Value": "John",
"Vendor": "Vendor1"
},
{
"Field": "PlayerNumber",
"Type": "23"
"Value": 2,
"Vendor": "Vendor1"
},
{
"Field": "ShortName",
"Type": "24"
"Value": "James",
"Vendor": "Vendor1"
}
]
I am trying to achieve this using JoltTransformJSON now. Is this achievable using JoltTransformJSON? I have it running on Nifi, if not possible with Jolt what are my options?
Related
I have create a connector to azureEventhub , it works fine to pull the data into the topic ,
My use case is to filter the messages that I m pullling basing on the message type .
Example :
{
"messageType": "Transformed",
"timeStamp": 1652113146105,
"payload": {
"externalId": "24323",
"equipmentType": "TemperatureSensor",
"measureType": "Temperature",
"normalizedData": {
"equipmentData": [
{
"key": "ReadingValue",
"value": 23,
"valueType": "number",
"measurementUnit": "celsius",
"measurementDateTime": "2022-05-09T16:18:34.0000000Z"
}
]
},
"dataProviderName": "LineMetrics"
},
},
{
"messageType": "IntegratorGenericDataStream",
"timeStamp": 1652113146103,
"payload": {
"dataSource": {
"type": "sensor",
},
"dataPoints": [
{
"type": "Motion",
"value": 0,
"valueType": "number",
"dateTime": "2022-05-09T16:18:37.0000000Z",
"unit": "count"
}
],
"dataProvider": {
"id": "ba84cbdb-cbf8-4d4f-9a55-93b43f671b5a",
"name": "LineMetrics",
"displayName": "Line Metrics"
}
},
}
I wanted to apply a filter on a value like shown in the pic:
enter image description here
the error that appears to me :
enter image description here
I try to filter some results data from mongodb with mongoose in javascript.
This is my json structure:
{
"name": "john",
"firstname": "doe",
"yearold": 22,
"recipes": [
{
"title": "cheesecake",
"data": [
{
"name": "egg",
"label": "Eggs for",
"value": 6,
"unit": "piece"
},
{
"name": "oil",
"label": "Specific oil",
"unit": "oz",
"value": 0.2
},
{
"name": "flour",
"label": "Wholemel flour",
"value": 450,
"unit": "gr"
}
]
},
{
"title": "cake",
"data": [
{
"name": "egg",
"label": "Eggs for",
"value": 6,
"unit": "piece"
},
{
"name": "flour",
"label": "Wholemel flour",
"value": 500,
"unit": "gr"
},
]
}
]
}
In some case i need to return json data with hiding some values. For example I have a list that specifies all the values to hide
hidekeys=["egg"];
and i would like to get this:
{
"name": "john",
"firstname": "doe",
"yearold": 22,
"recipes": [
{
"title": "cheesecake",
"data": [
{
"name": "egg",
"label": "Eggs for",
"value": #######,
"unit": "piece"
},
{
"name": "oil",
"label": "Specific oil",
"unit": "oz",
"value": 0.2
},
{
"name": "flour",
"label": "Wholemel flour",
"value": 450,
"unit": "gr"
}
]
},
{
"title": "cake",
"data": [
{
"name": "egg",
"label": "Eggs for",
"value": #######,
"unit": "piece"
},
{
"name": "flour",
"label": "Wholemel flour",
"value": 500,
"unit": "gr"
},
]
}
]
}
For each recipe i need to hide ingredient value if it is specified in hidekeys.
I tried something with $project and $cond but it doesnt works
Here's a quick way of how to achieve this using $map
const hidekeys = ["egg"];
db.collection.aggregate([
{
$addFields: {
recipes: {
$map: {
input: "$recipes",
as: "recipe",
in: {
$mergeObjects: [
"$$recipe",
{
data: {
$map: {
input: "$$recipe.data",
as: "datum",
in: {
"$mergeObjects": [
"$$datum",
{
$cond: [
{
"$setIsSubset": [
[
"$$datum.name"
],
hidekeys
]
},
{
value: "#####"
},
{
value: "$$datum.value"
}
]
}
]
}
}
}
}
]
}
}
}
}
}
])
Mongo Playground
I am trying filter the response city wise. I am not able to understand how to query filter parameters.
I have tried different ways but with no success. This is the response without applying filter. But I want filter it for a particular city.
{
"index_name": "3b01bcb8-0b14-4abf-b6f2-c1bfd384ba69",
"title": "Real time Air Quality Index from various location",
"desc": "Real time Air Quality Index from various location",
"org_type": "Central",
"org": [
"Ministry of Environment and Forests",
"Central Pollution Control Board"
],
"sector": [
"Industrial Air Pollution"
],
"source": "data.gov.in",
"catalog_uuid": "a3e7afc6-b799-4ede-b143-8e074b27e0621",
"visualizable": "1",
"active": "1",
"created": 1543320551,
"updated": 1559683085,
"created_date": "2018-11-27T17:39:11Z",
"updated_date": "2019-06-05T02:48:05Z",
"target_bucket": {
"index": "air_quality",
"type": "a3e7afc6-b799-4ede-b143-8e074b27e0621",
"field": "3b01bcb8-0b14-4abf-b6f2-c1bfd384ba69"
},
"field": [
{
"id": "id",
"name": "id",
"type": "double"
},
{
"id": "country",
"name": "country",
"type": "keyword"
},
{
"id": "state",
"name": "state",
"type": "keyword"
},
{
"id": "city",
"name": "city",
"type": "keyword"
},
{
"id": "station",
"name": "station",
"type": "keyword"
},
{
"id": "last_update",
"name": "last_update",
"type": "date"
},
{
"id": "pollutant_id",
"name": "pollutant_id",
"type": "keyword"
},
{
"id": "pollutant_min",
"name": "pollutant_min",
"type": "double"
},
{
"id": "pollutant_max",
"name": "pollutant_max",
"type": "double"
},
{
"id": "pollutant_avg",
"name": "pollutant_avg",
"type": "double"
},
{
"id": "pollutant_unit",
"name": "pollutant_unit",
"type": "keyword"
}
],
"status": "ok",
"message": "Resource detail",
"total": 1000,
"count": 10,
"limit": "10",
"offset": "8",
"records": [
{
"id": "13",
"country": "India",
"state": "Andhra_Pradesh",
"city": "Rajamahendravaram",
"station": "Anand Kala Kshetram, Rajamahendravaram - APPCB",
"last_update": "05-06-2019 02:00:00",
"pollutant_id": "CO",
"pollutant_min": "2",
"pollutant_max": "50",
"pollutant_avg": "28",
"pollutant_unit": "NA"
},
{
"id": "14",
"country": "India",
"state": "Andhra_Pradesh",
"city": "Rajamahendravaram",
"station": "Anand Kala Kshetram, Rajamahendravaram - APPCB",
"last_update": "05-06-2019 02:00:00",
"pollutant_id": "OZONE",
"pollutant_min": "37",
"pollutant_max": "132",
"pollutant_avg": "71",
"pollutant_unit": "NA"
}
{
"id": "16",
"country": "India",
"state": "Andhra_Pradesh",
"city": "Tirupati",
"station": "Tirumala, Tirupati - APPCB",
"last_update": "05-06-2019 02:00:00",
"pollutant_id": "PM10",
"pollutant_min": "33",
"pollutant_max": "72",
"pollutant_avg": "55",
"pollutant_unit": "NA"
}
],
"version": "2.1.0"
}
This is only documentation on how to do filtering.
properties: OrderedMap { "id": OrderedMap { "type": "integer" }, "date": OrderedMap { "type": "integer" } }
How to form request url for filtering the response?
yes the documentation is very poor, but still out of many trials I got it work like
this
https://api.data.gov.in/resource/3b01bcb8-0b14-4abf-b6f2-c1bfd384ba69?api-key=<your key>&format=json&offset=0&limit=10
&filters[pollutant_id]=NO2
My input JSON looks like below, but i am not sure of how to do internal array related parameter transformation using JOLT. Any help is appreciated as i am new to JOLT
{
"pktId": 7603,
"seq": 1,
"vehicleNumber": "66079",
"rmdLocation": "1",
"rmdTime": "2019-01-07T11:27:05.745Z",
"position": {
"lat": 55.4911232,
"lng": -3.686831
},
"dataSource": 11,
"frames": [
{
"seq": 0,
"card": 8,
"channel": 6,
"value": 117
},
{
"seq": 1,
"card": 8,
"channel": 6,
"value": 120
}
]
}
Below is the spec file i have created but it is not complete
[
{
"operation": "shift",
"spec": {
"frames": {
"*": {
"seq": "parameters[&1].seq",
"card": "parameters[&1].card",
"channel": "parameters[&1].channel",
"value": "parameters[&1].value"
}
},
"rmdTime": "messageTime",
"vehicleNumber": "roadNumber"
}
},
{
"operation": "default",
"spec": {
"appId": "configMsgXX",
"customerId": "ABC",
"messageRev": 1,
"messageType": "customStatistics"
}
}
]
Expected output is as below
{
"appId": "configMsgXX",
"customerId": "ABC",
"deviceId": string1+roadNumber+string2",
"messageRev": 1,
"messageTime": 1543395341000,
"messageType": "customStatistics",
"parameters": [
{
"address": string1+string2,
"name": "EM2000VoltageMainGenerator",
"timestamp": 1543395341000,
"quality": "3",
"datatype": "INTEGER",
"value": 100,
"qualityReason": "Stale Data",
"category": "REAL"
}
],
"roadNumber": 66079
}
I am using this library https://github.com/bazaarvoice/jolt
[
{
"operation": "shift",
"spec": {
"frames": {
"*": {
"seq": "parameters[&1].seq",
"card": "parameters[&1].card",
"channel": "parameters[&1].channel",
"value": "parameters[&1].value",
"#(2,rmdTime)": "parameters[&1].timestamp"
}
},
"rmdTime": "messageTime",
"vehicleNumber": "roadNumber"
}
},
{
"operation": "modify-overwrite-beta",
"spec": {
"parameters": {
"*": {
"quality": "3",
"name": "",
"address": "",
"datatype": "INTEGER",
"qualityReason": "Stale Data",
"category": "REAL"
}
}
}
},
{
"operation": "default",
"spec": {
"appId": "configMsgXX",
"customerId": "ABC",
"messageRev": 1,
"messageType": "customStatistics"
}
}
]
In this json , I need a find query which finds all the field where the "status":"Y", if the parent field has "status":"N", ignore the child field , else find the child field where the "status":"Y" along with its parent field
Note: The sub field is in a array
[
{
"type": "Type 1",
"status": "Y",
"code": "1",
"category": [
{
"type": "Cat 1",
"status": "Y",
"code": "1000",
"subcategories": [
{
"type": "Sub 1",
"status": "N",
"code": "1001"
},
{
"type": "Sub 2",
"status": "N",
"code": "1002"
},
{
"type": "Sub 3",
"status": "Y",
"code": "1003"
}
]
},
{
"type": "Cat 2",
"status": "N",
"code": "2000",
"subcategories": [
{
"type": "Sub 4",
"status": "Y",
"code": "2001"
},
{
"type": "Sub 5",
"status": "Y",
"code": "2002"
}
]
}
]
}
]
My Output Should be like this
[
{
"type": "Type 1",
"status": "Y",
"code": "1",
"category": [
{
"type": "Cat 1",
"status": "Y",
"code": "1000",
"subcategories": [
{
"type": "Sub 3",
"status": "Y",
"code": "1003"
}
]
} ]
}
]
Thanks in Advance:)
You can try below aggregation
db.collection.aggregate([
{ "$match": { "status": "Y" }},
{ "$unwind": "$category" },
{ "$match": { "category.status": "Y" } },
{ "$project": { "type": 1, "status": 1, "code": 1,
"category.type": "$category.type",
"category.status": "$category.status",
"category.code": "$category.code",
"category.subcategories": {
"$filter": {
"input": "$category.subcategories",
"as": "subcategory",
"cond": {
"$eq": [
"$$subcategory.status",
"Y"
]
}
}
}
}},
{ "$group": {
"_id": "$_id",
"type": { "$first": "$type" },
"status": { "$first": "$status" },
"code": { "$first": "$code" },
"category": { "$push": "$category" }
}}
]).then((data) => {
res.send(data)
})
Gives you following output (check here)
[
{
"_id": ObjectId("5a934e000102030405000000"),
"category": [
{
"code": "1000",
"status": "Y",
"subcategories": [
{
"code": "1003",
"status": "Y",
"type": "Sub 3"
}
],
"type": "Cat 1"
}
],
"code": "1",
"status": "Y",
"type": "Type 1"
}
]