MongoDB multiple fields lookup - mongodb

I'm trying to perform a $lookup using two fields on MongoDB 3.6. I've already read the docs and similar questions here, but I was unable to find what's wrong.
Collection acls:
[ { _id: 1, FolderId: 4, Sid: 'S-123-456' }
{ _id: 2, FolderId: 5, Sid: 'S-234-567' }
{ _id: 3, FolderId: 6, Sid: 'S-345-678' } ]
Collection groups:
[ { _id: 1, ProcessId: 10, Sid: 'S-123-456', Users: [ 'user1', 'user2'] }
{ _id: 2, ProcessId: 10, Sid: 'S-234-567', Users: [ 'user1'] }
{ _id: 3, ProcessId: 20, Sid: 'S-123-456', Users: [ 'user2'] } ]
Query:
db.acls.aggregate({
$lookup:
{
from: 'groups',
let: { 'ProcessId': 10, 'GroupSid': '$Sid' },
pipeline: [{
$match: {
$expr: {
$and: [
{
$eq: [ '$ProcessId', '$$ProcessId' ]
},
{
$eq: [ '$Sid', '$$GroupSid' ]
}
]
}
}
}],
as: 'grouplist'
}
})
I was expecting to return something like:
{ _id: 1, FolderId: 4, Sid: 'S-123-456',
grouplist: [ { _id: 1, ProcessId: 10, Sid: 'S-123-456', Users: [ 'user1', 'user2'] }] }
but instead I'm getting 'Script executed successfully, but there are no results to show', on Robo 3T.

Try This it's working fine. Your let keyword must be start with lowercase
db.acls.aggregate([
{
$lookup:
{
from: "groups",
let: { processid: 10, sid: "$Sid" },
pipeline: [
{ $match:
{ $expr:
{ $and:
[
{ $eq: [ "$ProcessId", "$$processid" ] },
{ $gte: [ "$Sid", "$$sid" ] }
]
}
}
}
],
as: "grouplist"
}
}
])

$let variable operator must start with the lower case letter.
db.acls.aggregate([
{ "$lookup": {
"from": 'groups',
"let": { "groupSid": "$Sid" },
"pipeline": [
{ "$match": {
"$expr": { "$eq": [ "$Sid", "$$groupSid" ] },
"ProcessId": 10
}}
],
"as": "grouplist"
}}
])

db.getCollection("acls").aggregate(
// Pipeline
[
// Stage 1
{
$lookup: // Equality Match
{
from: "groups",
localField: "Sid",
foreignField: "Sid",
as: "grouplist"
}
},
// Stage 2
{
$project: {
grouplist: {
$filter: {
input: "$grouplist",
as: "group",
cond: {
$eq: ["$$group.ProcessId", 10]
}
}
},
FolderId: 1,
Sid: 1
}
},
]
);

Related

Mongoose aggregate: how to create fields dynamically from the user request

Please someone help me! I can't find the solution in documentation or other topics.
I'm using mongodb aggregation in Mongoose/Nest.js project to return the document data with some formatting and filtering. I have the structure of the mongo document like
{
_id: '1',
outputs: [
{
fileName: 'fileName1',
data: [
{
columnName1: 3,
columnName2: 4,
........
columnName30: 5
},
{
columnName1: 1,
columnName2: 2,
........
columnName30: 3
},
...........
]
},
{
fileName: 'fileName1',
data: [
{
columnName1: 3,
columnName2: 4,
........
columnName30: 5
},
{
columnName1: 1,
columnName2: 2,
........
columnName30: 3
},
...........
]
}
........
]
}
I've already done some formatting, but now I need to include to the response only requested by the user fields (columnNamesToChoose). And filter their values depending on gte, lte of mainColumnName. Inside $project I was going to use some mapping like this, but it doesn't work. Could you please help me to fix this part of code?
...columnNamesToChoose.map((columnName) => ({ [columnName]: {
$map: {
input: {
$filter: {
input: '$outputs.data',
as: 'item',
cond: {
$and: [
{ $gte: [`$$item.${mainColumnName}`, gte] },
{ $lte: [`$$item.${mainColumnName}`, lte] },
],
},
},
},
as: 'file',
in: `$$file.${columnName}`,
},
} })),
This is the full code of aggregation:
mainColumnName = 'column1' (from the body of the user request)
columnNamesToChoose = ['column2', 'column5'] (from the body of the user request)
myModel.aggregate([
{
$match: { _id: Number(id) },
},
{ $unwind: '$outputs' },
{
$match: { 'outputs.fileName': fileName },
},
{
$project: {
_id: '$_id',
fileName: '$outputs.fileName',
[mainColumnName]: {
$map: {
input: {
$filter: {
input: '$outputs.data',
as: 'item',
cond: {
$and: [
{ $gte: [`$$item.${mainColumnName}`, gte] },
{ $lte: [`$$item.${mainColumnName}`, lte] },
],
},
},
},
as: 'file',
in: `$$file.${mainColumnName}`,
},
},
},
},
])
My result:
{
"0": {
"column2": [
4,
2,
1,
5
]
},
"1": {
"column5": [
1,
8,
9,
0
]
},
"_id": 1,
"fileName": "somefilename.txt",
"column1": [
3,
1,
2,
20
],
}
Expected result:
{
"_id": 1,
"fileName": "somefilename.txt",
"column1": [
3,
1,
2,
20
],
"column2": [
4,
2,
1,
5
],
"column5": [
1,
8,
9,
0
],
}
One option is to first $reduce and then $unwind, $match and $group, where the $group stage is built dynamically on the code (for-loop) according to the input:
db.collection.aggregate([
{$match: {_id: id}},
{$project: {
outputs: {
$reduce: {
input: "$outputs",
initialValue: [],
in: {
$concatArrays: [
"$$value",
{$cond: [
{$eq: ["$$this.fileName", fileName]},
"$$this.data",
[]
]
}
]
}
}
}
}
},
{$unwind: "$outputs"},
{$match: {"outputs.columnName1": {$gte: gte, $lte: lte}}},
{$group: {
_id: 0,
column1: {$push: "$outputs.columnName1"},
column2: {$push: "$outputs.columnName2"},
column5: {$push: "$outputs.columnName5"}
}},
{$set: {fileName: fileName}}
])
See how it works on the playground example
On js it will look something like:
const matchStage = {$match: {}};
matchStage.$match[`outputs.${mainColumnName}`] = {$gte: gte, $lte: lte};
const groupStage = {$group: {_id: 0}};
for (const col of columnNamesToChoose ) {
groupStage.$group[col] = {$push: `"$outputs.${col}"`}
};
const aggregation = [
{$match: {_id: id}},
{$project: {
outputs: {$reduce: {
input: "$outputs",
initialValue: [],
in: {$concatArrays: [
"$$value",
{$cond: [
{$eq: ["$$this.fileName", fileName]},
"$$this.data",
[]
]}
]}
}}
}},
{$unwind: "$outputs"},
matchStage,
groupStage,
{$set: {fileName: fileName}}
],
const res = await myModel.aggregate(aggregation)

MongoDB: Aggregation/Grouping for poll votes

Am trying to create a poll results aggregation
I have two collections
poll - here is one document
{
"_id": {
"$oid": "636027704f7a15587ef74f26"
},
"question": "question 1",
"ended": false,
"options": [
{
"id": "1",
"option": "option 1"
},
{
"id": "2",
"option": "option 2"
},
{
"id": "3",
"option": "option 3"
}
]
}
Vote - here is one document
{
"_id": {
"$oid": "635ed3210acbf9fd14af8fd1"
},
"poll_id": "636027704f7a15587ef74f26",
"poll_option_id": "1",
"user_id": "1"
}
and i want to perform an aggregate query to get poll results
so am doing the following query
db.vote.aggregate(
[
{
$addFields: {
poll_id: { "$toObjectId": "$poll_id" }
},
},
{
$lookup: {
from: "poll",
localField: "poll_id",
foreignField: "_id",
as: "details"
}
},
{
$group:
{
_id: { poll_id: "$poll_id", poll_option_id: "$poll_option_id" },
details: { $first: "$details" },
count: { $sum: 1 }
}
},
{
$addFields: {
question: { $arrayElemAt: ["$details.question", 0] }
}
},
{
$addFields: {
options: { $arrayElemAt: ["$details.options", 0] }
}
},
{
$group: {
_id: "$_id.poll_id",
poll_id: { $first: "$_id.poll_id" },
question: { $first: "$question" },
options: { $first: "$options" },
optionsGrouped: {
$push: {
id: "$_id.poll_option_id",
count: "$count"
}
},
count: { $sum: "$count" }
}
}
]
)
That is giving me this form of results
{ _id: ObjectId("636027704f7a15587ef74f26"),
poll_id: ObjectId("636027704f7a15587ef74f26"),
question: 'question 1',
options:
[ { id: '1', option: 'option 1' },
{ id: '2', option: 'option 2' },
{ id: '3', option: 'option 3' } ],
optionsGrouped:
[ { id: '1', count: 2 },
{ id: '2', count: 1 } ],
count: 3 }
So what am interested in i want to have the results looking like ( like merging both options & options Group)
{ _id: ObjectId("636027704f7a15587ef74f26"),
poll_id: ObjectId("636027704f7a15587ef74f26"),
question: 'question 1',
optionsGrouped:
[ { id: '1', option: 'option 1', count: 2 },
{ id: '2', option: 'option 2', count: 1 },
{ id: '3', option: 'option 3', count: 0 } ],
count: 4 }
Another question is the DB structure acceptable overall or i can represent that in a better way ?
One option is to group first and use the $lookup later, in order to fetch less data from the poll collection. After the $lookup, use $map with $cond to merge the arrays:
db.vote.aggregate([
{$group: {
_id: {poll_id: {$toObjectId: "$poll_id"}, poll_option_id: "$poll_option_id"},
count: {$sum: 1}
}},
{$group: {
_id: "$_id.poll_id",
counts: {
$push: {count: "$count", option: {$concat: ["option ", "$_id.poll_option_id"]}}
},
countAll: {$sum: "$count"}
}},
{$lookup: {
from: "poll",
localField: "_id",
foreignField: "_id",
as: "poll"
}},
{$project: {poll: {$first: "$poll"}, counts: 1, countAll: 1}},
{$project: {
optionsGrouped: {
$map: {
input: "$poll.options",
in: {$mergeObjects: [
"$$this",
{$cond: [
{$gte: [{$indexOfArray: ["$counts.option", "$$this.option"]}, 0]},
{$arrayElemAt: ["$counts", {$indexOfArray: ["$counts.option", "$$this.option"]}]},
{count: 0}
]}
]}
}
},
count: "$countAll",
question: "$poll.question"
}}
])
See how it works on the playground example
I had reworked the query to match my desires
and this query is achieving the question i have asked
db.poll.aggregate([
{
$addFields: {
_id: {
$toString: "$_id"
}
}
},
{
$lookup: {
from: "poll_vote",
localField: "_id",
foreignField: "poll_id",
as: "votes"
}
},
{
$replaceRoot: {
newRoot: {
$let: {
vars: {
count: {
$size: "$votes"
},
options: {
$map: {
input: "$options",
as: "option",
in: {
$mergeObjects: [
"$$option",
{
count: {
$size: {
$slice: [
{
$filter: {
input: "$votes",
as: "v",
cond: {
$and: [
{
$eq: [
"$$v.poll_option_id",
"$$option._id"
]
}
]
}
}
},
0,
100
]
}
}
},
{
checked: {
$toBool: {
$size: {
$slice: [
{
$filter: {
input: "$votes",
as: "v",
cond: {
$and: [
{
$eq: [
"$$v.user_id",
2
]
},
{
$eq: [
"$$v.poll_option_id",
"$$option._id"
]
}
]
}
}
},
0,
100
]
}
}
}
}
]
}
}
}
},
"in": {
_id: "$_id",
question: "$question",
count: "$$count",
ended: "$ended",
options: "$$options"
}
}
}
}
},
{
$addFields: {
answered: {
$reduce: {
input: "$options",
initialValue: false,
in: {
$cond: [
{
$eq: [
"$$this.checked",
true
]
},
true,
"$$value"
]
}
}
}
}
}
])

Merge array data and add some additional field by performing multiplication on field

I have below kind of schema
Mongo playground
Problem - I want to get all succeeded transaction in list with their respective user with some extra field like reward - for that particular transaction. like if paid amount in 10 then reward will be 0.3 times -> 3. But i need 0.3 in case of 1st successful payment only for others it will be 0.1.
tried: I have achieved partial output, not able to get reward calculation based on first transaction
The output will be -
[
{
_id: 1,
name: 'Stephen',
transactions: [
{
_id: 1,
paidAmount: 10,
reward: 3
},
{
_id: 3,
paidAmount: 20,
reward: 2
}
]
},
{
_id: 2,
user: 'Peter',
transactions: [
{
_id: 2,
paidAmount: 5,
reward: 0.15
}]
}
]
Hi i have found the solution for above
db.users.aggregate([
{
$match: {
_id: 1,
},
},
{
$lookup: {
from: "payments",
localField: "_id",
foreignField: "user",
as: "payments",
},
},
{
$addFields: {
transactions: {
$reduce: {
input: "$payments",
initialValue: [],
in: {
$concatArrays: [
"$$value",
{
$filter: {
input: "$$this.transactions",
cond: {
$eq: [
"$$this.status",
"succeeded"
]
},
},
},
],
},
},
},
},
},
{
$project: {
transactions: {
$map: {
input: "$transactions",
as: "t",
in: {
paidAmount: "$$t.paidAmount",
status: "$$t.status",
createdAt: "$$t.createdAt",
reward: {
$cond: {
if: {
$eq: [
{
$indexOfArray: [
"$transactions",
"$$t"
]
},
0
]
},
then: {
$multiply: [
"$$t.paidAmount",
0.3
]
},
else: {
$multiply: [
"$$t.paidAmount",
0.1
]
},
},
},
},
},
},
},
},
{
$project: {
id: {
$toString: "$_id"
},
totalAMount: {
$sum: "$transactions.paidAmount"
},
totalReward: {
$sum: "$transactions.reward"
},
transactions: "$transactions",
},
},
])
Please comment if you find out a better solution/ approach.
Try this

MongoDb query to exclude omission of rows based on criteria

In below example, looking for new partner suggestions for user abc. abc has already sent a request to 123 so that can be ignored. rrr has sent request to abc but rrr is in the fromUser field so rrr is still a valid row to be shown as suggestion to abc
I have two collections:
User collection
[
{
_id: "abc",
name: "abc",
group: 1
},
{
_id: "xyz",
name: "xyyy",
group: 1
},
{
_id: "123",
name: "yyy",
group: 1
},
{
_id: "rrr",
name: "tttt",
group: 1
},
{
_id: "eee",
name: "uuu",
group: 1
}
]
Partnership collection (if users have already partnered)
[
{
_id: "abc_123",
fromUser: "abc",
toUser: "123"
},
{
_id: "rrr_abc",
fromUser: "rrr",
toUser: "abc"
},
{
_id: "xyz_rrr",
fromUser: "xyz",
toUser: "rrr"
}
]
My query below excludes the user rrr but it should not because its not listed in toUser field in the partnership collection corresponding to the user abc.
How to modify this query to include user rrr in this case?
db.users.aggregate([
{
$match: {
group: 1,
_id: {
$ne: "abc"
}
}
},
{
$lookup: {
from: "partnership",
let: {
userId: "$_id"
},
as: "prob",
pipeline: [
{
$set: {
users: [
"$fromUser",
"$toUser"
],
u: "$$userId"
}
},
{
$match: {
$expr: {
$and: [
{
$in: [
"$$userId",
"$users"
]
},
{
$in: [
"abc",
"$users"
]
}
]
}
}
}
]
}
},
{
$match: {
"prob.0": {
$exists: false
}
}
},
{
$sample: {
size: 1
}
},
{
$unset: "prob"
}
])
https://mongoplayground.net/p/utGMeHFRGmt
Your current query does not allow creating an existing connection regardless of the connection direction. If the order of the connection is important use:
db.users.aggregate([
{$match: {
group: 1,
_id: {$ne: "abc"}
}
},
{$lookup: {
from: "partnership",
let: { userId: {$concat: ["abc", "_", "$_id"]}},
as: "prob",
pipeline: [{$match: {$expr: {$eq: ["$_id", "$$userId"]}}}]
}
},
{$match: {"prob.0": {$exists: false}}},
{$sample: {size: 1}},
{$unset: "prob"}
])
See how it works on the playground example
For MongoDB 5 and later, I'd propose the following aggregation pipeline:
db.users.aggregate([
{
$match: {
group: 1,
_id: {
$ne: "abc"
}
}
},
{
$lookup: {
from: "partnership",
as: "prob",
localField: "_id",
foreignField: "toUser",
pipeline: [
{
$match: {
fromUser: "abc",
}
}
]
}
},
{
$match: {
"prob.0": {
$exists: false
}
}
},
{
$unset: "prob"
}
])
The following documents are returned (full result without the $sample stage):
[
{
"_id": "eee",
"group": 1,
"name": "uuu"
},
{
"_id": "rrr",
"group": 1,
"name": "tttt"
},
{
"_id": "xyz",
"group": 1,
"name": "xyyy"
}
]
The main difference is that the lookup connects the collections by the toUser field (see localField, foreignField) and uses a minimal pipeline to restrict the results further to only retrieve the requests from the current user document to "abc".
See this playground to test.
When using MongoDB < 5, you cannot use localField and foreignField to run the pipeline only on a subset of the documents in the * from*
collection. To overcome this, you can use this aggregation pipeline:
db.users.aggregate([
{
$match: {
group: 1,
_id: {
$ne: "abc"
}
}
},
{
$lookup: {
from: "partnership",
as: "prob",
let: {
userId: "$_id"
},
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: [
"$fromUser",
"abc"
]
},
{
$eq: [
"$toUser",
"$$userId"
]
}
]
}
}
}
]
}
},
{
$match: {
"prob.0": {
$exists: false
}
}
},
{
$unset: "prob"
}
])
The results are the same as for the upper pipeline.
See this playground to test.
For another, another way, this query starts from the partnership collection, finds which users to exclude, and then does a "$lookup" for everybody else. The remainder is just output formatting, although it looks like you may want to add a "$sample" stage at the end.
db.partnership.aggregate([
{
"$match": {
"fromUser": "abc"
}
},
{
"$group": {
"_id": null,
"exclude": {"$push": "$toUser" }
}
},
{
"$lookup": {
"from": "users",
"let": {
"exclude": {"$concatArrays": [["abc"], "$exclude"]
}
},
"pipeline": [
{
"$match": {
"$expr": {
"$not": {"$in": ["$_id", "$$exclude"]}
}
}
}
],
"as": "output"
}
},
{
"$project": {
"_id": 0,
"output": 1
}
},
{"$unwind": "$output"},
{"$replaceWith": "$output"}
])
Try it on mongoplayground.net.

mongodb can $unionWith use in $push

I want to get related data based on current item processing.
Sample:
[
{ field1: 1, field2: 2, value: 12 },
{ field1: 1, field2: 2, value: 21 },
{ field1: 1, value: 1 },
{ field2: 2, value: 2 },
{ field1: 2, field2: 3, value: 23 }
];
and result:
[
{
_id: { field1: 1, field2: 2 },
value: [12, 12],
relatedValue: [1, 2], // of item 1 and 2 because field 1 = 1 or field 2 = 2
},
];
Sample query:
db.collectionA.aggregate([
{
$match: { field1: 1 }
},
{
"$group":{
"_id":{
"field1":"$field1",
"field2":"$field2"
},
"alerts":{
"$push":{
"_id":"$_id",
"value":"$value",
"relatedData": {
"$unionWith": {
"coll": "collectionA",
"pipeline": [{
"$match": {
"$or": [
{ "field1": "$field1" },
{ "field2": "$field2" }
]
}
}]
}
}
}
}
}
}
])
I tried run this query but error, Please help me fix or give a solution
// Edited: value should be array because I want to group data by field1, field2 and push all value of group to an array
You're trying to use $unionWith within $group but it is a "pipeline stage" meaning it can't be used like that, the same way you can't use $group within a $group.
Additionally this stage is used to "union" two collections and not to populate data based on value matches ( which it seems you're trying to do here ), for this case you want to use $lookup, like so:
db.collection.aggregate([
{
$lookup: {
from: "collection",
let: {
field1: "$field1",
field2: "$field2",
docId: "$_id"
},
pipeline: [
{
$match: {
$expr: {
$and: [
{
$or: [
{
$eq: [
"$$field1",
"$field1"
]
},
{
$eq: [
"$$field2",
"$field2"
]
}
]
},
{
$ne: [
"$$docId",
"$_id"
]
}
]
}
}
},
{
$project: {
value: 1
}
}
],
as: "relatedData"
}
},
{
$group: {
_id: {
field1: "$field1",
field2: "$field2"
},
values: {
$push: "$value"
},
relatedValue: {
$push: {
$map: {
input: "$relatedData",
in: "$$this.value"
}
}
}
}
},
{
$project: {
field1: "$_id.field1",
field2: "$_id.field2",
values: 1,
relatedValues: {
"$setDifference": [
{
"$reduce": {
input: "$relatedValue",
initialValue: [],
in: {
"$setUnion": [
"$$this",
"$$value"
]
}
}
},
"$values"
]
}
}
}
])
Mongo Playground