Issues with geo indexes in mongodb - mongodb

We have a collection of ~1M items that we query using the $nearSphere selector. It takes around between 3 seconds and 20 seconds to return 200 items.
From the explain plan of the request, we can see that is going through the same index 6 times.
Is it the expected behavior of mongodb query planner?
We would like to know if there is a way to force mongo to filter first by some field like endDate to reduce the set and then use $nearSphere?
On our monitoring system we can see some pagefault and assert but they might be related to the lack of IOPS of our hard drive.
Thank you for your help.
Here is the explan plan (I removed the rejected plans and troncates the BinData lines)
{
"queryPlanner": {
"plannerVersion": 1.0,
"namespace": "myCollection.Post",
"indexFilterSet": false,
"parsedQuery": {
"$and": [
{
"$or": [
{
"availableToUsers": {
"$eq": "M76zJCedq4"
}
},
{
"$nor": [
{
"availableToUsers": {
"$exists": true
}
}
]
}
]
},
{
"startDate": {
"$lt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"availableSubmitNumber": {
"$gt": 0.0
}
},
{
"endDate": {
"$gt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"name": {
"$in": ["Post1", "Post2"]
}
},
{
"$nor": [
{
"acceptedByUserId": {
"$eq": "M76zJCedq4"
}
}
]
},
{
"locationGeoPoint": {
"$nearSphere": [
174.9084055,
-36.9293289
]
}
}
]
},
"winningPlan": {
"stage": "FETCH",
"filter": {
"$and": [
{
"$or": [
{
"availableToUsers": {
"$eq": "M76zJCedq4"
}
},
{
"$nor": [
{
"availableToUsers": {
"$exists": true
}
}
]
}
]
},
{
"$nor": [
{
"acceptedByUserId": {
"$eq": "M76zJCedq4"
}
}
]
}
]
},
"inputStage": {
"stage": "GEO_NEAR_2D",
"keyPattern": {
"locationGeoPoint": "2d",
"endDate": 1.0,
"startDate": 1.0,
"availableSubmitNumber": 1.0,
"name": 1.0
},
"indexName": "locationGeoPoint_2d_endDate_1_startDate_1_availableSubmitNumber_1_name_1",
"indexVersion": 2.0,
"inputStages": [
{
"stage": "FETCH",
"inputStage": {
"stage": "IXSCAN",
"filter": {
"$and": [
{
"endDate": {
"$gt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"startDate": {
"$lt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"availableSubmitNumber": {
"$gt": 0.0
}
},
{
"name": {
"$in": ["Post1", "Post2"]
}
}
]
},
"keyPattern": {
"locationGeoPoint": "2d",
"endDate": 1.0,
"startDate": 1.0,
"availableSubmitNumber": 1.0,
"name": 1.0
},
"indexName": "locationGeoPoint_2d_endDate_1_startDate_1_availableSubmitNumber_1_name_1",
"isMultiKey": false,
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2.0,
"direction": "forward",
"indexBounds": {
"locationGeoPoint": [
"[BinData(128, BEB167B000000000), BinData(128, BEB167BFFFFFFFFF)]"
],
"endDate": [
"[MinKey, MaxKey]"
],
"startDate": [
"[MinKey, MaxKey]"
],
"availableSubmitNumber": [
"[MinKey, MaxKey]"
],
"name": [
"[MinKey, MaxKey]"
]
}
}
},
{
"stage": "FETCH",
"inputStage": {
"stage": "IXSCAN",
"filter": {
"$and": [
{
"endDate": {
"$gt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"startDate": {
"$lt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"availableSubmitNumber": {
"$gt": 0.0
}
},
{
"name": {
"$in": ["Post1", "Post2"]
}
}
]
},
"keyPattern": {
"locationGeoPoint": "2d",
"endDate": 1.0,
"startDate": 1.0,
"availableSubmitNumber": 1.0,
"name": 1.0
},
"indexName": "locationGeoPoint_2d_endDate_1_startDate_1_availableSubmitNumber_1_name_1",
"isMultiKey": false,
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2.0,
"direction": "forward",
"indexBounds": {
"locationGeoPoint": [
"[BinData(128, BEB1658000000000), BinData(128, BEB165BFFFFFFFFF)]",
"[BinData(128, BEB165C000000000), BinData(128, BEB165FFFFFFFFFF)]"
],
"endDate": [
"[MinKey, MaxKey]"
],
"startDate": [
"[MinKey, MaxKey]"
],
"availableSubmitNumber": [
"[MinKey, MaxKey]"
],
"name": [
"[MinKey, MaxKey]"
]
}
}
},
{
"stage": "FETCH",
"inputStage": {
"stage": "IXSCAN",
"filter": {
"$and": [
{
"endDate": {
"$gt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"startDate": {
"$lt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"availableSubmitNumber": {
"$gt": 0.0
}
},
{
"name": {
"$in": ["Post1", "Post2"]
}
}
]
},
"keyPattern": {
"locationGeoPoint": "2d",
"endDate": 1.0,
"startDate": 1.0,
"availableSubmitNumber": 1.0,
"name": 1.0
},
"indexName": "locationGeoPoint_2d_endDate_1_startDate_1_availableSubmitNumber_1_name_1",
"isMultiKey": false,
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2.0,
"direction": "forward",
"indexBounds": {
"locationGeoPoint": [
"[BinData(128, BEB14BC000000000), BinData(128, BEB14BFFFFFFFFFF)]",
"[BinData(128, BEB14C0000000000), BinData(128, BEB14FFFFFFFFFFF)]",
"[BinData(128, BEB1580000000000), BinData(128, BEB15BFFFFFFFFFF)]",
"[BinData(128, BEB1600000000000), BinData(128, BEB163FFFFFFFFFF)]",
"[BinData(128, BEB1640000000000), BinData(128, BEB164FFFFFFFFFF)]",
"[BinData(128, BEB1650000000000), BinData(128, BEB1653FFFFFFFFF)]",
"[BinData(128, BEB1654000000000), BinData(128, BEB1657FFFFFFFFF)]",
"[BinData(128, BEB1680000000000), BinData(128, BEB16BFFFFFFFFFF)]"
],
"endDate": [
"[MinKey, MaxKey]"
],
"startDate": [
"[MinKey, MaxKey]"
],
"availableSubmitNumber": [
"[MinKey, MaxKey]"
],
"name": [
"[MinKey, MaxKey]"
]
}
}
},
{
"stage": "FETCH",
"inputStage": {
"stage": "IXSCAN",
"filter": {
"$and": [
{
"endDate": {
"$gt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"startDate": {
"$lt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"availableSubmitNumber": {
"$gt": 0.0
}
},
{
"name": {
"$in": ["Post1", "Post2"]
}
}
]
},
"keyPattern": {
"locationGeoPoint": "2d",
"endDate": 1.0,
"startDate": 1.0,
"availableSubmitNumber": 1.0,
"name": 1.0
},
"indexName": "locationGeoPoint_2d_endDate_1_startDate_1_availableSubmitNumber_1_name_1",
"isMultiKey": false,
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2.0,
"direction": "forward",
"indexBounds": {
"locationGeoPoint": [
"[BinData(128, BE9BE00000000000), BinData(128, BE9BEFFFFFFFFFFF)]",
"[BinData(128, BE9BF80000000000), BinData(128, BE9BFBFFFFFFFFFF)]",
"[BinData(128, BEB1100000000000), BinData(128, BEB11FFFFFFFFFFF)]",
"[BinData(128, BEB1300000000000), BinData(128, BEB13FFFFFFFFFFF)]",
"[BinData(128, BEB1400000000000), BinData(128, BEB143FFFFFFFFFF)]"
],
"endDate": [
"[MinKey, MaxKey]"
],
"startDate": [
"[MinKey, MaxKey]"
],
"availableSubmitNumber": [
"[MinKey, MaxKey]"
],
"name": [
"[MinKey, MaxKey]"
]
}
}
},
{
"stage": "FETCH",
"inputStage": {
"stage": "IXSCAN",
"filter": {
"$and": [
{
"endDate": {
"$gt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"startDate": {
"$lt": ISODate(
"2019-03-01T01:02:00.000+0000"
)
}
},
{
"availableSubmitNumber": {
"$gt": 0.0
}
},
{
"name": {
"$in": ["Post1", "Post2"]
}
}
]
},
"keyPattern": {
"locationGeoPoint": "2d",
"endDate": 1.0,
"startDate": 1.0,
"availableSubmitNumber": 1.0,
"name": 1.0
},
"indexName": "locationGeoPoint_2d_endDate_1_startDate_1_availableSubmitNumber_1_name_1",
"isMultiKey": false,
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2.0,
"direction": "forward",
"indexBounds": {
"locationGeoPoint": [
"[BinData(128, BE9B800000000000), BinData(128, BE9BBFFFFFFFFFFF)]",
"[BinData(128, BE9BC00000000000), BinData(128, BE9BCFFFFFFFFFFF)]"
],
"endDate": [
"[MinKey, MaxKey]"
],
"startDate": [
"[MinKey, MaxKey]"
],
"availableSubmitNumber": [
"[MinKey, MaxKey]"
],
"name": [
"[MinKey, MaxKey]"
]
}
}
}
]
}
}
},
"serverInfo": {
"port": 27017.0,
"version": "4.0.3",
"gitVersion": "7ea530946fa7880364d88c8d8b6026bbc9ffa48c"
},
"ok": 1.0,
"operationTime": Timestamp(1551940718,
4),
"$clusterTime": {
"clusterTime": Timestamp(1551940718,
4),
"signature": {
"hash": BinData(0,
"AAAAAAAAAAAAAAAAAAAAAAAAAAA="
),
"keyId": NumberLong(0)
}
}
}

Related

MongoDB aggregation re-group deep nested array of objects

Good day all,
I need some help figuring out this aggregation issue. I have a document with nested arrays where I want to perform lookups on the nested array's values after which I want to restore the document to its original structure. see below:
[
{
"_id": "63c7fec2fe9afea23afdbcef",
"primary_language": "en",
"image_link": "avatar_image_linl",
"description": ["Some description text here"],
"source": "community",
"standard": "imperial",
"gender": "male",
"base_a": 47,
"base_w": 220,
"base_h": 71.65,
"status": "draft",
"date_created": "2023-01-18T14:14:26.201Z",
"product_plan": {
"p_len": 4,
"p_fre": 1,
"p_qua": 4,
"avg_len": 3583,
"total_wor": 16,
"total_exe": 128,
"plan": {
"field_we": 1,
"field_da": 1,
"field_ti": 1,
"field_rest_we": false,
"field_rest_da": false,
"product": {
"name": "W111",
"comment": {
"en": ""
},
"pro_diff": [
{
"itemID": 2,
"description": "Intermediate"
}
],
"pro_int": [
{
"itemID": 2,
"description": "Moderate"
}
],
"pro_dur": 3736,
"nbr_exe": 8,
"target_mus": [
{
"itemID": 1,
"description": "Adominorus"
},
{
"itemID": 16,
"description": "Tricerus"
},
{
"itemID": 5,
"description": "Chiwawak"
}
],
"target_are": [
{
"itemID": 1,
"description": "Adominorus"
},
{
"itemID": 4,
"description": "Chiwawak"
},
{
"itemID": 2,
"description": "Amanus"
}
],
"items": {
"itemid": 44,
"target_are": [
{
"itemID": 4,
"description": "Chiwawak"
}
],
"target_mus": [
{
"itemID": 5,
"description": "Chiwawak"
}
],
"nbr_ref": 4,
"int_rec": "06~10",
"type_rec": [
{
"itemID": 14,
"description": "Straight Set"
}
],
"tempo": "1-1-1-1",
"rest_time": 90,
"note": "",
"order": null,
"pro_dur": 520,
"_id": "63ca9b798d47745ae5589906",
"status": 0,
"pct_complete": 0,
"set": []
},
"isCustomName": false,
"status": 0,
"pct_complete": 0
},
"_id": "63ca95368d47745ae558985e"
},
"_id": "63c7fec2fe9afea23afdbcf0"
},
"title": "Short Title text here",
"itemID": "bodyfyme882212-1674051266.214",
"__v": 32
},
{
"_id": "63c7fec2fe9afea23afdbcef",
"primary_language": "en",
"image_link": "avatar_image_linl",
"description": ["Some description text here"],
"source": "community",
"standard": "imperial",
"gender": "male",
"base_a": 47,
"base_w": 220,
"base_h": 71.65,
"status": "draft",
"date_created": "2023-01-18T14:14:26.201Z",
"product_plan": {
"p_len": 4,
"p_fre": 1,
"p_qua": 4,
"avg_len": 3583,
"total_wor": 16,
"total_exe": 128,
"plan": {
"field_we": 1,
"field_da": 1,
"field_ti": 1,
"field_rest_we": false,
"field_rest_da": false,
"product": {
"name": "W111",
"comment": {
"en": ""
},
"pro_diff": [
{
"itemID": 2,
"description": "Intermediate"
}
],
"pro_int": [
{
"itemID": 2,
"description": "Moderate"
}
],
"pro_dur": 3736,
"nbr_exe": 8,
"target_mus": [
{
"itemID": 1,
"description": "Adominorus"
},
{
"itemID": 16,
"description": "Tricerus"
},
{
"itemID": 5,
"description": "Chiwawak"
}
],
"target_are": [
{
"itemID": 1,
"description": "Adominorus"
},
{
"itemID": 4,
"description": "Chiwawak"
},
{
"itemID": 2,
"description": "Amanus"
}
],
"items": {
"itemid": 339,
"target_are": [
{
"itemID": 4,
"description": "Chiwawak"
}
],
"target_mus": [
{
"itemID": 5,
"description": "Chiwawak"
}
],
"nbr_ref": 3,
"int_rec": "06~10",
"type_rec": [
{
"itemID": 14,
"description": "Straight Set"
}
],
"tempo": "1-1-1-1",
"rest_time": 90,
"note": "",
"order": null,
"pro_dur": 390,
"_id": "63ca9b798d47745ae5589907",
"status": 0,
"pct_complete": 0,
"set": []
},
"isCustomName": false,
"status": 0,
"pct_complete": 0
},
"_id": "63ca95368d47745ae558985e"
},
"_id": "63c7fec2fe9afea23afdbcf0"
},
"title": "Short Title text here",
"itemID": "bodyfyme882212-1674051266.214",
"__v": 32
},
{
"_id": "63c7fec2fe9afea23afdbcef",
"primary_language": "en",
"image_link": "avatar_image_linl",
"description": ["Some description text here"],
"source": "community",
"standard": "imperial",
"gender": "male",
"base_a": 47,
"base_w": 220,
"base_h": 71.65,
"status": "draft",
"date_created": "2023-01-18T14:14:26.201Z",
"product_plan": {
"p_len": 4,
"p_fre": 1,
"p_qua": 4,
"avg_len": 3583,
"total_wor": 16,
"total_exe": 128,
"plan": {
"field_we": 1,
"field_da": 2,
"field_ti": 1,
"field_rest_we": false,
"field_rest_da": false,
"product": {
"name": "W121",
"comment": {
"en": ""
},
"pro_diff": [
{
"itemID": 2,
"description": "Intermediate"
}
],
"pro_int": [
{
"itemID": 2,
"description": "Moderate"
}
],
"pro_dur": 2674,
"nbr_exe": 6,
"target_mus": [
{
"itemID": 9,
"description": "Lats"
},
{
"itemID": 10,
"description": "Lower Back"
},
{
"itemID": 11,
"description": "Middle Back"
}
],
"target_are": [
{
"itemID": 3,
"description": "Back"
}
],
"items": {
"itemid": 47,
"target_are": [
{
"itemID": 3,
"description": "Back"
}
],
"target_mus": [
{
"itemID": 10,
"description": "Lower Back"
}
],
"nbr_ref": 4,
"int_rec": "06~10",
"type_rec": [
{
"itemID": 14,
"description": "Straight Set"
}
],
"tempo": "1-1-1-1",
"rest_time": 90,
"note": "",
"order": null,
"pro_dur": 520,
"_id": "63caa1568d47745ae5589b25",
"status": 0,
"pct_complete": 0,
"set": []
},
"isCustomName": false,
"status": 0,
"pct_complete": 0
},
"_id": "63ca9b8e8d47745ae558994b"
},
"_id": "63c7fec2fe9afea23afdbcf0"
},
"title": "Short Title text here",
"itemID": "bodyfyme882212-1674051266.214",
"__v": 32
},
{
"_id": "63c7fec2fe9afea23afdbcef",
"primary_language": "en",
"image_link": "avatar_image_linl",
"description": ["Some description text here"],
"source": "community",
"standard": "imperial",
"gender": "male",
"base_a": 47,
"base_w": 220,
"base_h": 71.65,
"status": "draft",
"date_created": "2023-01-18T14:14:26.201Z",
"product_plan": {
"p_len": 4,
"p_fre": 1,
"p_qua": 4,
"avg_len": 3583,
"total_wor": 16,
"total_exe": 128,
"plan": {
"field_we": 1,
"field_da": 2,
"field_ti": 1,
"field_rest_we": false,
"field_rest_da": false,
"product": {
"name": "W121",
"comment": {
"en": ""
},
"pro_diff": [
{
"itemID": 2,
"description": "Intermediate"
}
],
"pro_int": [
{
"itemID": 2,
"description": "Moderate"
}
],
"pro_dur": 2674,
"nbr_exe": 6,
"target_mus": [
{
"itemID": 9,
"description": "Lats"
},
{
"itemID": 10,
"description": "Lower Back"
},
{
"itemID": 11,
"description": "Middle Back"
}
],
"target_are": [
{
"itemID": 3,
"description": "Back"
}
],
"items": {
"itemid": 495,
"target_are": [
{
"itemID": 3,
"description": "Back"
}
],
"target_mus": [
{
"itemID": 11,
"description": "Middle Back"
}
],
"nbr_ref": 3,
"int_rec": "10~12",
"type_rec": [
{
"itemID": 14,
"description": "Straight Set"
}
],
"tempo": "1-1-1-1",
"rest_time": 90,
"note": "",
"order": null,
"pro_dur": 414,
"_id": "63caa1568d47745ae5589b26",
"status": 0,
"pct_complete": 0,
"set": []
},
"isCustomName": false,
"status": 0,
"pct_complete": 0
},
"_id": "63ca9b8e8d47745ae558994b"
},
"_id": "63c7fec2fe9afea23afdbcf0"
},
"title": "Short Title text here",
"itemID": "bodyfyme882212-1674051266.214",
"__v": 32
},
{
......
}
]
Now i want to get the document back to this structure using aggregation.
{
"_id": "63c7fec2fe9afea23afdbcef",
"primary_language": "en",
"image_link": "avatar_image_linl",
"description": ["Some description text here"],
"source": "community",
"standard": "imperial",
"gender": "male",
"base_a": 47,
"base_w": 220,
"base_h": 71.65,
"status": "draft",
"date_created": "2023-01-18T14:14:26.201Z",
"product_plan": {
"p_len": 4,
"p_fre": 1,
"p_qua": 4,
"avg_len": 3583,
"total_wor": 16,
"total_exe": 128,
// Group plan here by product_plan._id
"plan": [
{
"field_we": 1,
"field_da": 1,
"field_ti": 1,
"field_rest_we": false,
"field_rest_da": false,
"product": {
"name": "W111",
"comment": {
"en": ""
},
"pro_diff": [
{
"itemID": 2,
"description": "Intermediate"
}
],
"pro_int": [
{
"itemID": 2,
"description": "Moderate"
}
],
"pro_dur": 3736,
"nbr_exe": 8,
"target_mus": [
{
"itemID": 1,
"description": "Adominorus"
},
{
"itemID": 16,
"description": "Tricerus"
},
{
"itemID": 5,
"description": "Chiwawak"
}
],
"target_are": [
{
"itemID": 1,
"description": "Adominorus"
},
{
"itemID": 4,
"description": "Chiwawak"
},
{
"itemID": 2,
"description": "Amanus"
}
],
// group items by plan._id
"items": [
{
"itemid": 44,
"target_are": [
{
"itemID": 4,
"description": "Chiwawak"
}
],
"target_mus": [
{
"itemID": 5,
"description": "Chiwawak"
}
],
"nbr_ref": 4,
"int_rec": "06~10",
"type_rec": [
{
"itemID": 14,
"description": "Straight Set"
}
],
"tempo": "1-1-1-1",
"rest_time": 90,
"note": "",
"order": null,
"pro_dur": 520,
"_id": "63ca9b798d47745ae5589906",
"status": 0,
"pct_complete": 0,
"set": []
},
{
"itemid": 339,
"target_are": [
{
"itemID": 4,
"description": "Chiwawak"
}
],
"target_mus": [
{
"itemID": 5,
"description": "Chiwawak"
}
],
"nbr_ref": 3,
"int_rec": "06~10",
"type_rec": [
{
"itemID": 14,
"description": "Straight Set"
}
],
"tempo": "1-1-1-1",
"rest_time": 90,
"note": "",
"order": null,
"pro_dur": 390,
"_id": "63ca9b798d47745ae5589907",
"status": 0,
"pct_complete": 0,
"set": []
},
],
"isCustomName": false,
"status": 0,
"pct_complete": 0
},
"_id": "63ca95368d47745ae558985e"
},
{
"field_we": 1,
"field_da": 2,
"field_ti": 1,
"field_rest_we": false,
"field_rest_da": false,
"product": {
"name": "W121",
"comment": {
"en": ""
},
"pro_diff": [
{
"itemID": 2,
"description": "Intermediate"
}
],
"pro_int": [
{
"itemID": 2,
"description": "Moderate"
}
],
"pro_dur": 2674,
"nbr_exe": 6,
"target_mus": [
{
"itemID": 9,
"description": "Lats"
},
{
"itemID": 10,
"description": "Lower Back"
},
{
"itemID": 11,
"description": "Middle Back"
}
],
"target_are": [
{
"itemID": 3,
"description": "Back"
}
],
// group items by plan._id
"items": [
{
"itemid": 47,
"target_are": [
{
"itemID": 3,
"description": "Back"
}
],
"target_mus": [
{
"itemID": 10,
"description": "Lower Back"
}
],
"nbr_ref": 4,
"int_rec": "06~10",
"type_rec": [
{
"itemID": 14,
"description": "Straight Set"
}
],
"tempo": "1-1-1-1",
"rest_time": 90,
"note": "",
"order": null,
"pro_dur": 520,
"_id": "63caa1568d47745ae5589b25",
"status": 0,
"pct_complete": 0,
"set": []
},
{
"itemid": 495,
"target_are": [
{
"itemID": 3,
"description": "Back"
}
],
"target_mus": [
{
"itemID": 11,
"description": "Middle Back"
}
],
"nbr_ref": 3,
"int_rec": "10~12",
"type_rec": [
{
"itemID": 14,
"description": "Straight Set"
}
],
"tempo": "1-1-1-1",
"rest_time": 90,
"note": "",
"order": null,
"pro_dur": 414,
"_id": "63caa1568d47745ae5589b26",
"status": 0,
"pct_complete": 0,
"set": []
}
],
"isCustomName": false,
"status": 0,
"pct_complete": 0
},
"_id": "63ca9b8e8d47745ae558994b"
}
],
"_id": "63c7fec2fe9afea23afdbcf0"
},
"title": "Short Title text here",
"itemID": "bodyfyme882212-1674051266.214",
"__v": 32
}
Any help would be greatly appreciated

Flutter Http response body from String to List of Object

Hi I am currently trying to fetch some data from an API, for later casting it to my Object Class.
The json answer i receive is instead of a list, directly a String.
{
"00-01": {
"date": "24-08-2022",
"hour": "00-01",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 617.5,
"units": "€/Mwh"
},
"01-02": {
"date": "24-08-2022",
"hour": "01-02",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 640.05,
"units": "€/Mwh"
},
"02-03": {
"date": "24-08-2022",
"hour": "02-03",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 670.26,
"units": "€/Mwh"
},
"03-04": {
"date": "24-08-2022",
"hour": "03-04",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 683.64,
"units": "€/Mwh"
},
"04-05": {
"date": "24-08-2022",
"hour": "04-05",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 692.88,
"units": "€/Mwh"
},
"05-06": {
"date": "24-08-2022",
"hour": "05-06",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 681.87,
"units": "€/Mwh"
},
"06-07": {
"date": "24-08-2022",
"hour": "06-07",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 624.35,
"units": "€/Mwh"
},
"07-08": {
"date": "24-08-2022",
"hour": "07-08",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 624.82,
"units": "€/Mwh"
},
"08-09": {
"date": "24-08-2022",
"hour": "08-09",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 623.24,
"units": "€/Mwh"
},
"09-10": {
"date": "24-08-2022",
"hour": "09-10",
"is-cheap": false,
"is-under-avg": true,
"market": "PVPC",
"price": 558.55,
"units": "€/Mwh"
},
"10-11": {
"date": "24-08-2022",
"hour": "10-11",
"is-cheap": false,
"is-under-avg": true,
"market": "PVPC",
"price": 511.3,
"units": "€/Mwh"
},
"11-12": {
"date": "24-08-2022",
"hour": "11-12",
"is-cheap": false,
"is-under-avg": true,
"market": "PVPC",
"price": 493.36,
"units": "€/Mwh"
},
"12-13": {
"date": "24-08-2022",
"hour": "12-13",
"is-cheap": true,
"is-under-avg": true,
"market": "PVPC",
"price": 484.42,
"units": "€/Mwh"
},
"13-14": {
"date": "24-08-2022",
"hour": "13-14",
"is-cheap": true,
"is-under-avg": true,
"market": "PVPC",
"price": 487.58,
"units": "€/Mwh"
},
"14-15": {
"date": "24-08-2022",
"hour": "14-15",
"is-cheap": true,
"is-under-avg": true,
"market": "PVPC",
"price": 426.72,
"units": "€/Mwh"
},
"15-16": {
"date": "24-08-2022",
"hour": "15-16",
"is-cheap": true,
"is-under-avg": true,
"market": "PVPC",
"price": 418.28,
"units": "€/Mwh"
},
"16-17": {
"date": "24-08-2022",
"hour": "16-17",
"is-cheap": true,
"is-under-avg": true,
"market": "PVPC",
"price": 422.18,
"units": "€/Mwh"
},
"17-18": {
"date": "24-08-2022",
"hour": "17-18",
"is-cheap": true,
"is-under-avg": true,
"market": "PVPC",
"price": 430.63,
"units": "€/Mwh"
},
"18-19": {
"date": "24-08-2022",
"hour": "18-19",
"is-cheap": false,
"is-under-avg": true,
"market": "PVPC",
"price": 495.26,
"units": "€/Mwh"
},
"19-20": {
"date": "24-08-2022",
"hour": "19-20",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 579.65,
"units": "€/Mwh"
},
"20-21": {
"date": "24-08-2022",
"hour": "20-21",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 614.08,
"units": "€/Mwh"
},
"21-22": {
"date": "24-08-2022",
"hour": "21-22",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 625.97,
"units": "€/Mwh"
},
"22-23": {
"date": "24-08-2022",
"hour": "22-23",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 582.99,
"units": "€/Mwh"
},
"23-24": {
"date": "24-08-2022",
"hour": "23-24",
"is-cheap": false,
"is-under-avg": false,
"market": "PVPC",
"price": 617.25,
"units": "€/Mwh"
}
}
I would like to cast that result into a LightHours, where every one would be a class Hour inside.
My main ideas was to get a List<LightHours> = {Hours,...}
Also I have tried to do the following: LightPrice lightPrice = LightPrice.fromJson(jsonDecode(response.body)); but it only gets me every field as null.
But using the json to Dart converter it doesn't allow me.
How could i convert that response into both objects classes?
Something like this maybe works:
List<LightPrice> list = (jsonDecode(response.body) as Map<String,
Map<String, dynamic>>).values.map<LightPrice>((value) =>
LightPrice.fromJson(value)).toList();
//your json string
String jsonString = json.encode(data);
//convert json string to list
List<String> newData = List<String>.from(json.decode(jsonString));
this will help to convert

MongoDB: use $sum with multiple $cond

I want calculate points based to the size of an array element.
I can use $sum with multiple $cond?
Something like this:
{
$project: {
points: {
$sum: [
{
$cond: [
{
$and: [{ $gt: [{ $arrayElemAt: ["$matches", 0] }, 0] }, { $gt: [{ $arrayElemAt: ["$matches", 1] }, 0] }, { $gt: [{ $arrayElemAt: ["$matches", 2] }, 0] }],
},
10,
0,
],
},
{
$cond: [
{
$and: [{ $gt: [{ $arrayElemAt: ["$matches", 3] }, 0] }, { $gt: [{ $arrayElemAt: ["$matches", 4] }, 0] }, { $gt: [{ $arrayElemAt: ["$matches", 5] }, 0] }],
},
10,
0,
],
},
],
},
},
},
This script always returns 20 even if not all conditions are true
Example of document:
{
"_id" : NumberInt(5),
"total_matches" : NumberInt(0),
"matches" : [
[
],
[
],
[
],
[
],
[
],
[
],
[
],
[
],
[
]
]
}
OUTPUT
{
"_id" : NumberInt(5),
"points" : 20.0,
"matches" : [
[
],
[
],
[
],
[
],
[
],
[
],
[
],
[
],
[
]
]
}
Ok, the problem is that I forgot $size in the query.
The right query is this:
{
$project: {
points: {
$sum: [
{
$cond: [
{
$and: [{ $gt: [{ $size:{ $arrayElemAt: ["$matches", 0] }}, 0] }, { $gt: [{ $arrayElemAt: ["$matches", 1] }, 0] }, { $gt: [{ $arrayElemAt: ["$matches", 2] }, 0] }],
},
10,
0,
],
},
{
$cond: [
{
$and: [{ $gt: [{ $size:{ $arrayElemAt: ["$matches", 3] }}, 0] }, { $gt: [{ $arrayElemAt: ["$matches", 4] }, 0] }, { $gt: [{ $arrayElemAt: ["$matches", 5] }, 0] }],
},
10,
0,
],
},
],
},
},
}

Merge object fields aggregation

I'm using MongoDB 3.6 and I have the following data:
[
{
"valoresVencimentos": [
1468.12,
85.9,
14.1,
1899.99,
241.92,
869.99,
696.11
],
"classesVencimentos": [
"lim_up_1ano",
"lim_1ano",
"a_venc_180d",
"a_venc_180d",
"outros",
"a_venc_180d",
"a_venc_180d"
],
"classesModalidades": [
"financiamento",
"financiamento",
"curto prazo",
"cartao",
"cartao",
"cartao",
"cartao"
]
},
{
"valoresVencimentos": [
627.29,
241.92,
413.47,
229.74,
1687.58,
100
],
"classesVencimentos": [
"a_venc_180d",
"outros",
"a_venc_180d",
"a_venc_180d",
"lim_up_1ano",
"lim_1ano"
],
"classesModalidades": [
"cartao",
"cartao",
"cartao",
"cartao",
"financiamento",
"financiamento"
]
},
{
"valoresVencimentos": [
268.59,
27.6,
428.51,
173.85,
2301.45,
100,
241.11
],
"classesVencimentos": [
"a_venc_180d",
"outros",
"a_venc_180d",
"a_venc_180d",
"lim_up_1ano",
"lim_1ano",
"a_venc_180d"
],
"classesModalidades": [
"cartao",
"cartao",
"cartao",
"cartao",
"financiamento",
"financiamento",
"curto prazo"
]
}
]
And I need merge fields with same name of each object in the input array, to be:
{
"valoresVencimentos": [
1468.12,
85.9,
14.1,
1899.99,
241.92,
869.99,
696.11,
627.29,
241.92,
413.47,
229.74,
1687.58,
100,
268.59,
27.6,
428.51,
173.85,
2301.45,
100,
241.11
],
"classesVencimentos": [
"lim_up_1ano",
"lim_1ano",
"a_venc_180d",
"a_venc_180d",
"outros",
"a_venc_180d",
"a_venc_180d",
"a_venc_180d",
"outros",
"a_venc_180d",
"a_venc_180d",
"lim_up_1ano",
"lim_1ano",
"a_venc_180d",
"outros",
"a_venc_180d",
"a_venc_180d",
"lim_up_1ano",
"lim_1ano",
"a_venc_180d"
],
"classesModalidades": [
"financiamento",
"financiamento",
"curto prazo",
"cartao",
"cartao",
"cartao",
"cartao",
"cartao",
"cartao",
"cartao",
"cartao",
"financiamento",
"financiamento",
"cartao",
"cartao",
"cartao",
"cartao",
"financiamento",
"financiamento",
"curto prazo"
]
}
Currently my aggregation is:
db.collection.aggregate([
{
$match: { code: '11122233344' }
},
{
$project: {
_id: 0,
valoresVencimentos: '$response.operations.expirations.value',
classesVencimentos: '$response.operations.expirations.class',
classesModalidades: '$response.operations.expirations.modality'
}
},
{
$addFields: {
valoresVencimentos: {
$reduce: {
input: '$valoresVencimentos',
initialValue: [],
in: { $concatArrays: ['$$this', '$$value'] }
}
},
classesVencimentos: {
$reduce: {
input: '$classesVencimentos',
initialValue: [],
in: { $concatArrays: ['$$this', '$$value'] }
}
},
classesModalidades: {
$reduce: {
input: '$classesModalidades',
initialValue: [],
in: { $concatArrays: ['$$this', '$$value'] }
}
},
}
},
])
It's imporant to say that ordering should be the same of the input data, so I think that using $group maybe a problem, I already tried a lot of alternatives but my knowledge with the aggregation framework is kind limited, appreciate any help.
Try this:
db.myCollection.aggregate([
{
$facet: {
valoresVencimentos: [
{ $unwind: "$valoresVencimentos" },
{
$group: {
_id: null,
valoresVencimentos: { $push: "$valoresVencimentos" }
}
}
],
classesVencimentos: [
{ $unwind: "$classesVencimentos" },
{
$group: {
_id: null,
classesVencimentos: { $push: "$classesVencimentos" }
}
}
],
classesModalidades: [
{ $unwind: "$classesModalidades" },
{
$group: {
_id: null,
classesModalidades: { $push: "$classesModalidades" }
}
}
]
}
},
{ $unwind: "$valoresVencimentos" },
{ $unwind: "$classesVencimentos" },
{ $unwind: "$classesModalidades" },
{
$project: {
valoresVencimentos: "$valoresVencimentos.valoresVencimentos",
classesVencimentos: "$classesVencimentos.classesVencimentos",
classesModalidades: "$classesModalidades.classesModalidades"
}
}
]);
or you can combine last three $unwinds and $project into single $project:
...,
{
$project: {
valoresVencimentos: { $arrayElemAt: ["$valoresVencimentos.valoresVencimentos", 0] },
classesVencimentos: { $arrayElemAt: ["$classesVencimentos.classesVencimentos", 0] },
classesModalidades: { $arrayElemAt: ["$classesModalidades.classesModalidades", 0] }
}
}
...

How to calculate YTD and MTD in mongodb?

How to calculate Month-To-Date(MTD) and Year-To-Date(YTD) in mongodb in a single query? sample data below, in this data requestedOn is a date field, I want to calculate MTD & YTD, on the assumption of financial year on "1st Jan of the year"(For example financial year for year 2016 is "01-Jan-2016" :
{
"_id": {
"$oid": "5808578b33fa6f161c9747f8"
},
"_class": "exceltest.TestBean",
"requestedOn": "2000-03-01",
"bookName": "Test6",
"revenue": 10.0,
"unitsSold": 1,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
},
{
"categoryCode": "Cooking/Beverages/Bartending"
},
{
"categoryCode": "Food Receipe/Taste"
}
]
}{
"_id": {
"$oid": "5808578b33fa6f161c9747f9"
},
"_class": "exceltest.TestBean",
"requestedOn": "2000-03-01",
"bookName": "Test1",
"revenue": 11.0,
"unitsSold": 2,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
},
{
"categoryCode": "Cooking/Beverages/Bartending"
},
{
"categoryCode": "Food Receipe/Taste"
}
]
}{
"_id": {
"$oid": "5808578b33fa6f161c9747fa"
},
"_class": "exceltest.TestBean",
"requestedOn": "2000-06-01",
"bookName": "Test2",
"revenue": 12.0,
"unitsSold": 3,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
},
{
"categoryCode": "Cooking/Beverages/Bartending"
},
{
"categoryCode": "Food Receipe/Taste"
}
]
}{
"_id": {
"$oid": "5808578b33fa6f161c9747fb"
},
"_class": "exceltest.TestBean",
"requestedOn": "2000-07-01",
"bookName": "Test3",
"revenue": 13.0,
"unitsSold": 4,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
},
{
"categoryCode": "Cooking/Beverages/Bartending"
},
{
"categoryCode": "Food Receipe/Taste"
}
]
}{
"_id": {
"$oid": "5808578b33fa6f161c9747fc"
},
"_class": "exceltest.TestBean",
"requestedOn": "2009-09-01",
"bookName": "Test4",
"revenue": 14.0,
"unitsSold": 5,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
},
{
"categoryCode": "Cooking/Beverages/Bartending"
},
{
"categoryCode": "Food Receipe/Taste"
}
]
}{
"_id": {
"$oid": "5808578b33fa6f161c9747fd"
},
"_class": "exceltest.TestBean",
"requestedOn": "2009-06-01",
"bookName": "Test5",
"revenue": 15.0,
"unitsSold": 6,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
},
{
"categoryCode": "Cooking/Beverages/Bartending"
},
{
"categoryCode": "Food Receipe/Taste"
}
]
}{
"_id": {
"$oid": "5808578b33fa6f161c9747fe"
},
"_class": "exceltest.TestBean",
"requestedOn": "2004-06-01",
"bookName": "Test10",
"revenue": 16.0,
"unitsSold": 7,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
},
{
"categoryCode": "Cooking/Beverages/Bartending"
},
{
"categoryCode": "Food Receipe/Taste"
}
]
}{
"_id": {
"$oid": "5808578b33fa6f161c9747ff"
},
"_class": "exceltest.TestBean",
"requestedOn": "2000-01-01",
"bookName": "Test11",
"revenue": 100.0,
"unitsSold": 100,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
},
{
"categoryCode": "Cooking/Beverages/Bartending"
},
{
"categoryCode": "Food Receipe/Taste"
}
]
}{
"_id": {
"$oid": "580857b833fa6f0c3499e462"
},
"_class": "exceltest.TestBean",
"requestedOn": "2000-02-01",
"bookName": "Test1",
"revenue": 20.0,
"unitsSold": 10,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
}
]
}{
"_id": {
"$oid": "580857b833fa6f0c3499e463"
},
"_class": "exceltest.TestBean",
"requestedOn": "2001-02-01",
"bookName": "Test2",
"revenue": 19.0,
"unitsSold": 9,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
}
]
}{
"_id": {
"$oid": "580857b833fa6f0c3499e464"
},
"_class": "exceltest.TestBean",
"requestedOn": "2001-02-01",
"bookName": "Test3",
"revenue": 18.0,
"unitsSold": 8,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
}
]
}{
"_id": {
"$oid": "580857b833fa6f0c3499e465"
},
"_class": "exceltest.TestBean",
"requestedOn": "2007-06-01",
"bookName": "Test4",
"revenue": 17.0,
"unitsSold": 7,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
}
]
}{
"_id": {
"$oid": "580857b833fa6f0c3499e466"
},
"_class": "exceltest.TestBean",
"requestedOn": "2005-06-01",
"bookName": "Test5",
"revenue": 16.0,
"unitsSold": 6,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
}
]
}{
"_id": {
"$oid": "580857b833fa6f0c3499e467"
},
"_class": "exceltest.TestBean",
"requestedOn": "2004-06-01",
"bookName": "Test1",
"revenue": 15.0,
"unitsSold": 5,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
}
]
}{
"_id": {
"$oid": "580857b833fa6f0c3499e468"
},
"_class": "exceltest.TestBean",
"requestedOn": "2002-06-01",
"bookName": "Test2",
"revenue": 14.0,
"unitsSold": 4,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
}
]
}{
"_id": {
"$oid": "580857b833fa6f0c3499e469"
},
"_class": "exceltest.TestBean",
"requestedOn": "2001-06-01",
"bookName": "Test3",
"revenue": 13.0,
"unitsSold": 3,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
}
]
}{
"_id": {
"$oid": "580857b833fa6f0c3499e46a"
},
"_class": "exceltest.TestBean",
"requestedOn": "2000-06-01",
"bookName": "Test4",
"revenue": 12.0,
"unitsSold": 2,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
}
]
}{
"_id": {
"$oid": "580857b833fa6f0c3499e46b"
},
"_class": "exceltest.TestBean",
"requestedOn": "2008-06-01",
"bookName": "Test5",
"revenue": 11.0,
"unitsSold": 1,
"bookCategory": [
{
"categoryCode": "Cooking/"
},
{
"categoryCode": "Cooking/Beverages"
},
{
"categoryCode": "Food Receipe/"
},
{
"categoryCode": "Food Receipe/Bartending"
}
]
}
Regards
Kris
It is a good practice to keep dates in MongoDB in its native dateformat ISODate().
You can use date formats like $year,$month,$day,$hour etc.
These can be used for grouping , in your case:
db.collectionName.aggregate([
{$group:{_id:{'Date':{$year:'$requestedOn'}},total:{$sum:'$FieldName'}}}
])
to convert string to ISODate , answers can be found at
- [http://stackoverflow.com/questions/15473772/how-to-convert-from-string-to-date-data-type?noredirect=1&lq=1][2]
- [http://stackoverflow.com/questions/15473772/how-to-convert-from-string-to-date-data-type?noredirect=1&lq=1][2]