Combining read and write into one query with mongodb - mongodb

I have two mongodb collections items and settings.
When documents are inserted into items, one of the fields level needs to be read from the settings table. Currently I'm running two queries to make this happen.
Is there a way to do this as one query like I could do with SQL with select queries inside an insert?
// read
let settingRecord = connection.collection("settings").findOne({
"group_id": 2
}, function(err, result) {});
// write
connection.collection("items").insertOne({
"item_id": 2,
"group_id": 2,
"level": settingRecord['level']
}, function(err, result) {});

You can do as below using $merge but with some more pipelines as mentioned by #D. SM
db.getCollection('test').aggregate( [
{ $match : { group: 2 } },
{ $project: {level: 1, group:1} },
{ $addFields:{item:2} },
{ $merge : { into: { db: "local", coll: "test2" }, whenMatched: [{ $addFields: {
"level":"$$new.level", "group":"$$new.group", "item":"$$new.item"
} } ] , whenNotMatched: "insert" }}
] )
Explanation:
{ $match : { group: 2 } }
Match on a field to get the document - In your case, it would be settings collection
{ $project: {level: 1, group:1} },
Project only the required fields to the next stage. You can skip this stage.
{ $addFields:{item:2} },
Using this you can add list of new fields to be inserted.
Actual magic happens in $merge
into - db and collection where data has to be inserted
whenMatched - add what fields to be inserted, $$new is used to refer the fields matched in the previous stage
Refer this
You can play with the pipelines to avoid unnecessary things. $merge has more options. In this example, you can skip whenNotMatched if you don't want to insert for non match items. You can specify to fail in that case.

I could merge into a new collection with a merge command for the given requirement of updating the level field. Let me know how to update the same collection as in aggregate collection eg. items collection in this case.
> db.newCollection.find();
{ "_id" : 1, "groupid" : 2, "itemid" : 2, "settingsDoc" : [ { "level" : 10 } ] }
{ "_id" : 2, "groupid" : 3, "itemid" : 3, "settingsDoc" : [ { "level" : 30 } ] }
> db.items1.find();
{ "_id" : 1, "groupid" : 2, "itemid" : 2, "level" : null }
{ "_id" : 2, "groupid" : 3, "itemid" : 3, "level" : null }
> db.settings.find();
{ "_id" : 1, "groupid" : 2, "level" : 10 }
{ "_id" : 2, "groupid" : 3, "level" : 30 }
> db.items1.aggregate([
... {$lookup:{
... from: "settings",
... localField: "groupid",
... foreignField: "groupid",
... as:"settingsDoc"
... }
... },
... {$project:{
... groupid:1,
... itemid:1,
... "settingsDoc.level":1
... }
... },
... {$merge:{into:"newCollection"}
... }
... ]);
> db.newCollection.find();
{ "_id" : 1, "groupid" : 2, "itemid" : 2, "settingsDoc" : [ { "level" : 10 } ] }
{ "_id" : 2, "groupid" : 3, "itemid" : 3, "settingsDoc" : [ { "level" : 30 } ] }
>

Related

MongoDB grouping aggregate query with relationship

Let's say that I have the following documents in the association collection:
{
"id" : 1,
"parentId" : 1,
"position" : {
"x" : 1,
"y" : 1
},
"tag" : "Beta"
},
{
"id" : 2,
"parentId" : 2,
"position" : {
"x" : 2,
"y" : 2
},
"tag" : "Alpha"
},
{
"id" : 3,
"parentId" : 1,
"position" : {
"x" : 3,
"y" : 3
},
"tag" : "Delta"
},
{
"id" : 4,
"parentId" : 1,
"position" : {
"x" : 4,
"y" : 4
},
"tag" : "Gamma"
},
{
"id" : 5,
"parentId" : 2,
"position" : {
"x" : 5,
"y" : 6
},
"tag" : "Epsilon"
}
I would like to create an aggregate query to produce the following result:
{
"_id" : 2,
"position" : {
"x" : 2,
"y" : 2
},
"tag" : "Alpha",
"children" : [
{
"position" : {
"x" : 5,
"y" : 6
},
"tag" : "Epsilon"
}
]
},
{
"_id" : 1,
"position" : {
"x" : 1,
"y" : 1
},
"tag" : "Beta"
"children" : [
{
"position" : {
"x" : 3,
"y" : 3
},
"tag" : "Delta"
},
{
"position" : {
"x" : 4,
"y" : 4
},
"tag" : "Gamma"
}
]
}
However, I was able only to create the following grouping query which puts "all-the-related" documents in children array:
db.association.aggregate([{
$group : {
_id : "$parentId",
children : {
$push : {
position : "$position",
tag :"$tag"
}
}
}
}])
I don't know how to filter out "position" and "tag" specific to "parent" points and put them at the top level.
Although Valijon's answer is working, it needs to be sorted before.
Here's a solution without the need of sorting, but using graphLookup stage (which is perfect to achieve what you need)
db.collection.aggregate([
{
$graphLookup: {
from: "collection",
startWith: "$id",
connectFromField: "id",
connectToField: "parentId",
as: "children",
}
},
{
$match: {
$expr: {
$gt: [
{
$size: "$children"
},
0
]
}
}
},
{
$addFields: {
children: {
$filter: {
input: "$children",
as: "child",
cond: {
$ne: [
"$id",
"$$child.id"
]
}
}
}
}
}
])
The first stage is doing the job.
The second one is here to filter documents that don't have any child.
The third is present only to remove parent from children array. But if you can remove self-reference in the parent, this last stage will not be needed anymore.
You can try it here
By making sure the documents are sorted (parent - children 1 - children 2 ... - children n), we can merge grouped document with the 1st child (which is parent). In the last step, we need to remove parent from children array.
Try this one:
db.association.aggregate([
{
$sort: {
parentId: 1,
id: 1
}
},
{
$group: {
_id: "$parentId",
children: {
$push: {
position: "$position",
tag: "$tag"
}
}
}
},
{
$replaceRoot: {
newRoot: {
$mergeObjects: [
"$$ROOT",
{
$arrayElemAt: [
"$children",
0
]
}
]
}
}
},
{
$addFields: {
children: {
$slice: [
"$children",
1,
{
$size: "$children"
}
]
}
}
}
])
MongoPlayground

How can I do match after second level unwind in mongodb?

I am working on a software that uses MongoDB as a database. I have a collection like this (this is just one document)
{
"_id" : ObjectId("5aef51e0af42ea1b70d0c4dc"),
"EndpointId" : "89799bcc-e86f-4c8a-b340-8b5ed53caf83",
"DateTime" : ISODate("2018-05-06T19:05:04.574Z"),
"Url" : "test",
"Tags" : [
{
"Uid" : "E2:02:00:18:DA:40",
"Type" : 1,
"DateTime" : ISODate("2018-05-06T19:05:04.574Z"),
"Sensors" : [
{
"Type" : 1,
"Value" : NumberDecimal("-98")
},
{
"Type" : 2,
"Value" : NumberDecimal("-65")
}
]
},
{
"Uid" : "12:3B:6A:1A:B7:F9",
"Type" : 1,
"DateTime" : ISODate("2018-05-06T19:05:04.574Z"),
"Sensors" : [
{
"Type" : 1,
"Value" : NumberDecimal("-95")
},
{
"Type" : 2,
"Value" : NumberDecimal("-59")
},
{
"Type" : 3,
"Value" : NumberDecimal("12.939770381907275")
}
]
}
]
}
and I want to run this query on it.
db.myCollection.aggregate([
{ $unwind: "$Tags" },
{
$match: {
$and: [
{
"Tags.DateTime": {
$gte: ISODate("2018-05-06T19:05:02Z"),
$lte: ISODate("2018-05-06T19:05:09Z"),
},
},
{ "Tags.Uid": { $in: ["C1:3D:CA:D4:45:11"] } },
],
},
},
{ $unwind: "$Tags.Sensors" },
{ $match: { "$Tags.Sensors.Type": { $in: [1, 2] } } },
{
$project: {
_id: 0,
EndpointId: "$EndpointId",
TagId: "$Tags.Uid",
Url: "$Url",
TagType: "$Tags.Type",
Date: "$Tags.DateTime",
SensorType: "$Tags.Sensors.Type",
Value: "$Tags.Sensors.Value",
},
},
])
the problem is, the second match (that checks $Tags.Sensors.Type) doesn't work and doesn't affect the result of the query.
How can I solve that?
If this is not the right way, what is the right way to run these conditions?
The $match stage accepts field names without a leading $ sign. You've done that correctly in your first $match stage but in the second one you write $Tags.Sensors.Type. Simply removing the leading $ sign should make your query work.
Mind you, the whole thing can be a bit simplified (and some beautification doesn't hurt, either):
You don't need to use $and in your example since it's assumed by default if you specify more than one criterion in a filter.
The $in that you use for the Tags.Sensors.Type filter can be a simple : kind of equality operator unless you have more than one element in the list of acceptable values.
In the $project stage, instead of (kind of) duplicating identical field names you can use the <field>: 1 syntax unless the order of the fields matters.
So the final query would be something like this.
db.myCollection.aggregate([
{
"$unwind" : "$Tags"
},
{
"$match" : {
"Tags.DateTime" : { "$gte" : ISODate("2018-05-06T19:05:02Z"), "$lte" : ISODate("2018-05-06T19:05:09Z") },
"Tags.Uid" : { "$in" : ["C1:3D:CA:D4:45:11"] }
}
}, {
"$unwind" : "$Tags.Sensors"
}, {
"$match" : {
"Tags.Sensors.Type" : { "$in" : [1,2] }
}
},
{
"$project" : {
"_id" : 0,
"EndpointId" : 1,
"TagId" : "$Tags.Uid",
"Url" : 1,
"TagType" : "$Tags.Type",
"Date" : "$Tags.DateTime",
"SensorType" : "$Tags.Sensors.Type",
"Value" : "$Tags.Sensors.Value"
}
}])

Return first result within each result of an `IN` query in Mongo

I am running an IN query against my collection.
Here is my query structure:
db.myCollection.find (
{ deviceId : { $in : [ "ABC", "XYZ" .. ] }
})
I know for a fact that each of these will return me multiple rows, but I am only interested in getting the first result of each.
I looked into the $first aggregation function, but could not see it a fit for my situation.
I could do a findOne query against each ID one at a time and combine the results in my client code written in Java. But there can be a lot of these IDs and I want to reduce the network round trip each time.
EDIT. Adding an example for more clarity.
Sample data in my collection:
{ "_id" : 1, "deviceSerial" : 1, "deviceId" : "ABC" }
{ "_id" : 2, "deviceSerial" : 2, "deviceId" : "XYZ" }
{ "_id" : 3, "deviceSerial" : 3, "deviceId" : "LMN" }
{ "_id" : 4, "deviceSerial" : 4, "deviceId" : "PQR" }
{ "_id" : 5, "deviceSerial" : 5, "deviceId" : "SDS" }
{ "_id" : 6, "deviceSerial" : 6, "deviceId" : "KLP" }
Now, if I do my query with { deviceId : { $in : [ "LMN", "XYZ" ] }
Expected Output (sort does not matter):
{ "_id" : 2, "deviceSerial" : 2, "deviceId" : "XYZ" }
{ "_id" : 3, "deviceSerial" : 3, "deviceId" : "LMN" }
So the idea with $first is good. You need to filter the collection with $in and then eliminate duplicates. Following aggregation should work:
db.myCollection.aggregate([
{
$match: {
deviceId: { $in: ["ABC", "XYZ"] }
}
},
{
$group: {
_id: "$deviceId",
doc: { "$first": "$$CURRENT" }
}
},
{
$replaceRoot: { newRoot: "$doc" }
}
])
doc will store first entire document for each group. On the last stage we need to promote this document to be a root and $replaceRoot is able to do that.

MongoDB filtering out subdocuments with lookup aggregation

Our project database has a capped collection called values which gets updated every few minutes with new data from sensors. These sensors all belong to a single sensor node, and I would like to query the last data from these nodes in a single aggregation. The problem I am having is filtering out just the last of ALL the types of sensors while still having only one (efficient) query. I looked around and found the $group argument, but I can't seem to figure out how to use it correctly in this case.
The database is structured as follows:
nodes:
{
"_id": 681
"sensors": [
{
"type": "foo"
},
{
"type": "bar"
}
]
}
values:
{
"_id" : ObjectId("570cc8b6ac55850d5740784e"),
"timestamp" : ISODate("2016-04-12T12:06:46.344Z"),
"type" : "foo",
"nodeid" : 681,
"value" : 10
}
{
"_id" : ObjectId("190ac8b6ac55850d5740776e"),
"timestamp" : ISODate("2016-04-12T12:06:46.344Z"),
"type" : "bar",
"nodeid" : 681,
"value" : 20
}
{
"_id" : ObjectId("167bc997bb66750d5740665e"),
"timestamp" : ISODate("2016-04-12T12:06:46.344Z"),
"type" : "bar",
"nodeid" : 200,
"value" : 20
}
{
"_id" : ObjectId("110cc9c6ac55850d5740784e"),
"timestamp" : ISODate("2016-04-09T12:06:46.344Z"),
"type" : "foo",
"nodeid" : 681,
"value" : 12
}
so let's imagine I want the data from node 681, I would want a structure like this:
nodes:
{
"_id": 681
"sensors": [
{
"_id" : ObjectId("570cc8b6ac55850d5740784e"),
"timestamp" : ISODate("2016-04-12T12:06:46.344Z"),
"type" : "foo",
"nodeid" : 681,
"value" : 10
},
{
"_id" : ObjectId("190ac8b6ac55850d5740776e"),
"timestamp" : ISODate("2016-04-12T12:06:46.344Z"),
"type" : "bar",
"nodeid" : 681,
"value" : 20
}
]
}
Notice how one value of foo is not queried, because I want to only get the latest value possible if there are more than one value (which is always going to be the case). The ordering of the collection is already according to the timestamp because the collection is capped.
I have this query, but it just gets all the values from the database (which is waaay too much to do in a lifetime, let alone one request of the web app), so I was wondering how I would filter it before it gets aggregated.
query:
db.nodes.aggregate(
[
{
$unwind: "$sensors"
},
{
$match:{
nodeid: 681
}
},
{
$lookup:{
from: "values", localField: "sensors.type", foreignField: "type", as: "sensors"
}
}
}
]
)
Try this
// Pipeline
[
// Stage 1 - sort the data collection if not already done (optional)
{
$sort: {
"timestamp":1
}
},
// Stage 2 - group by type & nodeid then get first item found in each group
{
$group: {
"_id":{type:"$type",nodeid:"$nodeid"},
"sensors": {"$first":"$$CURRENT"} //consider using $last if your collection is on reverse
}
},
// Stage 3 - project the fields in desired
{
$project: {
"_id":"$sensors._id",
"timestamp":"$sensors.timestamp",
"type":"$sensors.type",
"nodeid":"$sensors.nodeid",
"value":"$sensors.value"
}
},
// Stage 4 - group and push it to array sensors
{
$group: {
"_id":{nodeid:"$nodeid"},
"sensors": {"$addToSet":"$$CURRENT"}
}
}
]
as far as I got document structure, there is no need to use $lookup as all data is in readings(values) collection.
Please see proposed solution:
db.readings.aggregate([{
$match : {
nodeid : 681
}
},
{
$group : {
_id : {
type : "$type",
nodeid : "$nodeid"
},
readings : {
$push : {
timestamp : "$timestamp",
value : "$value",
id : "$_id"
}
}
}
}, {
$project : {
_id : "$_id",
readings : {
$slice : ["$readings", -1]
}
}
}, {
$unwind : "$readings"
}, {
$project : {
_id : "$readings.id",
type : "$_id.type",
nodeid : "$_id.nodeid",
timestamp : "$readings.timestamp",
value : "$readings.value",
}
}, {
$group : {
_id : "$nodeid",
sensors : {
$push : {
_id : "$_id",
timestamp : "$timestamp",
value : "$value",
type:"$type"
}
}
}
}
])
and output:
{
"_id" : 681,
"sensors" : [
{
"_id" : ObjectId("110cc9c6ac55850d5740784e"),
"timestamp" : ISODate("2016-04-09T12:06:46.344Z"),
"value" : 12,
"type" : "foo"
},
{
"_id" : ObjectId("190ac8b6ac55850d5740776e"),
"timestamp" : ISODate("2016-04-12T12:06:46.344Z"),
"value" : 20,
"type" : "bar"
}
]
}
Any comments welcome!

Use $project aggregation to edit documents schema - MongoDB

I have this MongoDB document:
{
"_id" : 1,
title: "abc123",
isbn: "0001122223334",
author: { last: "zzz", first: "aaa" },
copies: 5
}
Using $project (aggregation operator) I have to reshape the previous schema to get:
{
"_id" : 1,
field: {key:"title", value:"abc123"},
isbn: "0001122223334",
author: { last: "zzz", first: "aaa" },
copies: 5
}
To reach my target I used the following aggregation:
db.book.aggregate([{$project: {"field": {"key":"title", "value":"$title"}}}])
but I got an error:
{
"ok" : 0,
"errmsg" : "FieldPath 'isbn' doesn't start with $",
"code" : 16873
} : aggregate failed
I don't understand why that aggregation does not work, since if I want to reshape the previous schema to get:
{
"_id" : 1,
"author" : {
"last" : "zzz",
"first" : "aaa"
},
"copies" : 5,
"fieldTitle" : {
"key" : "abc123"
}
}
I can use this aggregation (and it works):
db.book.aggregate([{$project: {_id:1, fieldTitle:{key:"$title"}, author:1, copies:1}}])
Use the $literal operator to return a value without parsing. It's used for values that the aggregation pipeline may interpret as an expression, like the error you are presently getting:
db.book.aggregate([
{
$project: {
"field.key": { "$literal": "title" },
"field.value": "$title",
"author": 1, "copies": 1
}
}
])
Sample Output
{
"result" : [
{
"_id" : 1,
"author" : {
"last" : "zzz",
"first" : "aaa"
},
"copies" : 5,
"field" : {
"key" : "title",
"value" : "abc123"
}
}
],
"ok" : 1
}