How to take the duplicate records in mongodb - mongodb

Here i have two document ,in this documents childNodes array ID is duplicate means , i want to take the userID and pedagogyID of the record,as per my documents second document under childNodes array 798 is coming duplicate, so i want to take the records
Documents
{
"userID" : "A",
"pedagogyID" : "100",
"summary" : {
"LearnProgress" : {
"childNodes" : [
{
"ID" : "123",
"status" : "in-progress"
},
{
"ID" : "456",
"status" : null
},
{
"ID" : "333",
"status" : null
}
],
}
}
}
{
"userID" : "B",
"pedagogyID" : "200",
"summary" : {
"LearnProgress" : {
"childNodes" : [
{
"ID" : "789",
"status" : "in-progress"
},
{
"ID" : "1010",
"status" : null
},
{
"ID" : "789",
"status" : null
}
],
}
}
}
Expected Output
{
"userID" : "B",
"pedagogyID" : "200",
}
MY Code
db.collectionname.aggregate(
[
{"$unwind":"$summary.LearnProgress.childNodes"},
{"$group":{
"_id":{"_id":"$_id","ID":"$summary.LearnProgress.childNodes.ID"},
"userID":{"$first":"$userID"},
"pedagogyID":{"$first":"$pedagogyID"},
"count":{"$sum":1}
}},
{"$match":{"count":{"$gt":1}}},
{"$group":{"_id":{"userID":"$userID","pedagogyID":"$pedagogyID"}}},
{"$replaceRoot":{"newRoot":"$_id"}}
],
{ allowDiskUse:true }
)

You can use below aggregation.
db.colname.aggregate([
{"$unwind":"$summary.LearnProgress.childNodes"},
{"$group":{
"_id":{"_id":"$_id","ID":"$summary.LearnProgress.childNodes.ID"},
"userID":{"$first":"$userID"},
"pedagogyID":{"$first":"$pedagogyID"},
"count":{"$sum":1}
}},
{"$match":{"count":{"$gt":1}}},
{"$group":{"_id":{"userID":"$userID","pedagogyID":"$pedagogyID"}}},
{"$replaceRoot":{"newRoot":"$_id"}}
],{"allowDiskUse":true})

db.collectionname.aggregate(
// Pipeline
[
// Stage 1
{
$unwind: {
path : "$summary.LearnProgress.childNodes",
}
},
// Stage 2
{
$group: {
_id:'$summary.LearnProgress.childNodes.ID',
count:{$sum:1},
pedagogyID:{$first:'$pedagogyID'},
userID:{$first:'$userID'}
}
},
// Stage 3
{
$match: {
count:{$gt:1}
}
},
// Stage 4
{
$project: {
userID:1,
pedagogyID:1,
_id:0
}
},
]
);

Related

Partition data around a match query during aggregation

What I have been trying to get my head around is to perform some kind of partitioning(split by predicate) in a mongo query. My current query looks like:
db.posts.aggregate([
{"$match": { $and:[ {$or:[{"toggled":false},{"toggled":true, "status":"INACTIVE"}]} , {"updatedAt":{$gte:1549786260000}} ] }},
{"$unwind" :"$interests"},
{"$group" : {"_id": {"iid": "$interests", "pid":"$publisher"}, "count": {"$sum" : 1}}},
{"$project":{ _id: 0, "iid": "$_id.iid", "pid": "$_id.pid", "count": 1 }}
])
This results in the following output:
{
"count" : 3.0,
"iid" : "INT456",
"pid" : "P789"
}
{
"count" : 2.0,
"iid" : "INT789",
"pid" : "P789"
}
{
"count" : 1.0,
"iid" : "INT123",
"pid" : "P789"
}
{
"count" : 1.0,
"iid" : "INT123",
"pid" : "P123"
}
All good so far, but then I had realized that for the documents that match the specific filter {"toggled":true, "status":"INACTIVE"}, I would rather decrement the count (-1). (considering the eventual value can be negative as well.)
Is there a way to somehow partition the data after match to make sure different grouping operations are performed for both the collection of documents?
Something that sounds similar to what I am looking for is
$mergeObjects, or maybe $reduce, but not much that I can relate from the documentation examples.
Note: I can sense, one straightforward way to deal with this would be to perform two queries, but I am looking for a single query to perform the operation.
Sample documents for the above output would be:
/* 1 */
{
"_id" : ObjectId("5d1f7******"),
"id" : "CON123",
"title" : "Game",
"content" : {},
"status" : "ACTIVE",
"toggle":false,
"publisher" : "P789",
"interests" : [
"INT456"
],
"updatedAt" : NumberLong(1582078628264)
}
/* 2 */
{
"_id" : ObjectId("5d1f8******"),
"id" : "CON456",
"title" : "Home",
"content" : {},
"status" : "INACTIVE",
"toggle":true,
"publisher" : "P789",
"interests" : [
"INT456",
"INT789"
],
"updatedAt" : NumberLong(1582078628264)
}
/* 3 */
{
"_id" : ObjectId("5d0e9******"),
"id" : "CON654",
"title" : "School",
"content" : {},
"status" : "ACTIVE",
"toggle":false,
"publisher" : "P789",
"interests" : [
"INT123",
"INT456",
"INT789"
],
"updatedAt" : NumberLong(1582078628264)
}
/* 4 */
{
"_id" : ObjectId("5d207*******"),
"id" : "CON789",
"title":"Stack",
"content" : { },
"status" : "ACTIVE",
"toggle":false,
"publisher" : "P123",
"interests" : [
"INT123"
],
"updatedAt" : NumberLong(1582078628264)
}
What I am looking forward to as a result though is
{
"count" : 1.0, (2-1)
"iid" : "INT456",
"pid" : "P789"
}
{
"count" : 0.0, (1-1)
"iid" : "INT789",
"pid" : "P789"
}
{
"count" : 1.0,
"iid" : "INT123",
"pid" : "P789"
}
{
"count" : 1.0,
"iid" : "INT123",
"pid" : "P123"
}
This aggregation gives the desired result.
db.posts.aggregate( [
{ $match: { updatedAt: { $gte: 1549786260000 } } },
{ $facet: {
FALSE: [
{ $match: { toggle: false } },
{ $unwind : "$interests" },
{ $group : { _id : { iid: "$interests", pid: "$publisher" }, count: { $sum : 1 } } },
],
TRUE: [
{ $match: { toggle: true, status: "INACTIVE" } },
{ $unwind : "$interests" },
{ $group : { _id : { iid: "$interests", pid: "$publisher" }, count: { $sum : -1 } } },
]
} },
{ $project: { result: { $concatArrays: [ "$FALSE", "$TRUE" ] } } },
{ $unwind: "$result" },
{ $replaceRoot: { newRoot: "$result" } },
{ $group : { _id : "$_id", count: { $sum : "$count" } } },
{ $project:{ _id: 0, iid: "$_id.iid", pid: "$_id.pid", count: 1 } }
] )
[ EDIT ADD ]
The output from the query using the input data from the question post:
{ "count" : 1, "iid" : "INT123", "pid" : "P789" }
{ "count" : 1, "iid" : "INT123", "pid" : "P123" }
{ "count" : 0, "iid" : "INT789", "pid" : "P789" }
{ "count" : 1, "iid" : "INT456", "pid" : "P789" }
[ EDIT ADD 2 ]
This query gets the same result with different approach (code):
db.posts.aggregate( [
{
$match: { updatedAt: { $gte: 1549786260000 } }
},
{
$unwind : "$interests"
},
{
$group : {
_id : {
iid: "$interests",
pid: "$publisher"
},
count: {
$sum: {
$switch: {
branches: [
{ case: { $eq: [ "$toggle", false ] },
then: 1 },
{ case: { $and: [ { $eq: [ "$toggle", true] }, { $eq: [ "$status", "INACTIVE" ] } ] },
then: -1 }
]
}
}
}
}
},
{
$project:{
_id: 0,
iid: "$_id.iid",
pid: "$_id.pid",
count: 1
}
}
] )
[ EDIT ADD 3 ]
NOTE:
The facet query runs the two facets (TRUE and FALSE) on the same set of documents; it is like two queries running in parallel. But, there is some duplication of code as well as additional stages for shaping the documents down the pipeline to get the desired output.
The second query avoids the code duplication, and there are much lesser stages in the aggregation pipeline. This will make difference when the input dataset has a large number of documents to process - in terms of performance. In general, lesser stages means lesser iterations of the documents (as a stage has to scan the documents which are output from the previous stage).

monogdb nested array items exact match

I have a collection as below what I want is to fetch the items that has exact match of Tag="dolore", I tried different ways but I am getting all the elements if any of the embedded element has tag as dolore
{
"_id" : 123,
"vendor" : "ut",
"boxes" : [
{
"boxRef" : 321,
"items" : [
{
"Tag" : "dolore",
},
{
"Tag" : "irure",
},
{
"Tag" : "labore",
}
]
},
{
"boxRef" : 789,
"items" : [
{
"Tag" : "incididunt",
},
{
"Tag" : "magna",
},
{
"Tag" : "laboris",
}
]
},
{
"boxRef" : 456,
"items" : [
{
"Tag" : "reprehenderit",
},
{
"Tag" : "reprehenderit",
},
{
"Tag" : "enim",
}
]
}
]
}
If you are expecting to get only the matching embedded documents you have $unwind, $match and then $group to reverse the $unwind. Like this:
db.getCollection('collectionName').aggregate([
{
$unwind:"$boxes"
},
{
$unwind:"$boxes.items"
},
{
$match:{
"boxes.items.Tag":"dolore"
}
},
{
$group:{
_id:{
boxRef:"$boxes.boxRef",
_id:"$_id"
},
vendor:{
"$first":"$vendor"
},
boxRef:{
"$first":"$boxes.boxRef"
},
items:{
$push:"$boxes.items"
}
}
},
{
$group:{
_id:"$_id._id",
vendor:{
"$first":"$vendor"
},
boxes:{
$push:{
boxRef:"$boxRef",
items:"$items"
}
}
}
},
])
Output:
{
"_id" : 123.0,
"vendor" : "ut",
"boxes" : [
{
"boxRef" : 321.0,
"items" : [
{
"Tag" : "dolore"
}
]
}
]
}

mongodb find the document by id and then group the result based on name field

I have a collection with multiple documents like
{
"_id" : ObjectId("5a64d076bfd103df081967ae"),
"status" : "",
"Number" : 53,
"values" : [
{
"date" : "2015-05-18",
"value" : 12.41
},
{
"date" : "2015-05-19",
"value" : 12.45
},
],
"Name" : "ABC Banking",
"scheme":"ABC1",
"createdDate" : "21-01-2018"
}
{
"_id" : ObjectId("5a64d076bfd103df081967ae"),
"status" : "",
"Number" : 53,
"values" : [
{
"date" : "2015-05-18",
"value" : 13.41
},
{
"date" : "2015-05-19",
"value" : 13.45
},
],
"Name" : "ABC Banking",
"scheme":"ABC2",
"createdDate" : "21-01-2018"
}
I am Querying collection based on Number field like
db.getCollection('mfhistories').find({'Number':53})
to get all the documents with this Number.
Now I want to group all the collection with Name 'ABC Banking' into an array. so that I will get result based on Name.
so the result should be like
{
"Name":"ABC Banking",
[
{
"_id" : ObjectId("5a64d076bfd103df081967ae"),
"status" : "",
"Number" : 53,
"values" : [
{
"date" : "2015-05-18",
"value" : 13.41
},
{
"date" : "2015-05-19",
"value" : 13.45
},
],
"scheme":"ABC1",
"createdDate" : "21-01-2018"
},
{
"_id" : ObjectId("5a64d076bfd103df081967ae"),
"status" : "",
"Number" : 53,
"values" : [
{
"date" : "2015-05-18",
"value" : 13.41
},
{
"date" : "2015-05-19",
"value" : 13.45
}
],
"scheme":"ABC2",
"createdDate" : "21-01-2018"
}
]
}
Please help..
Thanks,
J
You can use Aggregation Framework for that:
db.col.aggregate([
{
$match: { Number: 53, Name: "ABC Banking" }
},
{
$group: {
_id: "$Name",
docs: { $push: "$$ROOT" }
}
},
{
$project: {
Name: "$_id",
_id: 0,
docs: 1
}
}
])
$$ROOT is a special variable which captures entire document. More here.
db.mfhistories.aggregate(
// Pipeline
[
// Stage 1
{
$match: {
Number: 53
}
},
// Stage 2
{
$group: {
_id: {
Name: '$Name'
},
docObj: {
$addToSet: '$$CURRENT'
}
}
},
// Stage 3
{
$project: {
Name: '$_id.Name',
docObj: 1,
_id: 0
}
}
]
);

mongodb aggregation $group and then $push a object

this is my data :
> db.bookmarks.find({"userId" : "56b9b74bf976ab70ff6b9999"}).pretty()
{
"_id" : ObjectId("56c2210fee4a33579f4202dd"),
"userId" : "56b9b74bf976ab70ff6b9999",
"items" : [
{
"itemId" : "28",
"timestamp" : "2016-02-12T18:07:28Z"
},
{
"itemId" : "29",
"timestamp" : "2016-02-12T18:07:29Z"
},
{
"itemId" : "30",
"timestamp" : "2016-02-12T18:07:30Z"
},
{
"itemId" : "31",
"timestamp" : "2016-02-12T18:07:31Z"
},
{
"itemId" : "32",
"timestamp" : "2016-02-12T18:07:32Z"
},
{
"itemId" : "33",
"timestamp" : "2016-02-12T18:07:33Z"
},
{
"itemId" : "34",
"timestamp" : "2016-02-12T18:07:34Z"
}
]
}
I want to have something like (actually i hope the _id can become userId too) :
{
"_id" : "56b9b74bf976ab70ff6b9999",
"items" : [
{ "itemId": "32", "timestamp": "2016-02-12T18:07:32Z" },
{ "itemId": "31", "timestamp": "2016-02-12T18:07:31Z" },
{ "itemId": "30", "timestamp": "2016-02-12T18:07:30Z" }
]
}
What I have now :
> db.bookmarks.aggregate(
... { $match: { "userId" : "56b9b74bf976ab70ff6b9999" } },
... { $unwind: '$items' },
... { $sort: { 'items.timestamp': -1} },
... { $skip: 2 },
... { $limit: 3},
... { $group: { '_id': '$userId' , items: { $push: '$items.itemId' } } }
... ).pretty()
{ "_id" : "56b9b74bf976ab70ff6b9999", "items" : [ "32", "31", "30" ] }
i tried to read the document in mongo and find out i can $push, but somehow i cannot find a way to push such object, which is not defined anywhere in the whole object. I want to have the timestamp also.. but i don't know how should i modified the $group (or others??) to do so. thanks for helping!
This code, which I tested in the MongoDB 3.2.1 shell, should give you the output format that you want:
> db.bookmarks.aggregate(
{ "$match" : { "userId" : "Ursula" } },
{ "$unwind" : "$items" },
{ "$sort" : { "items.timestamp" : -1 } },
{ "$skip" : 2 },
{ "$limit" : 3 },
{ "$group" : { "_id" : "$userId", items: { "$push" : { "myPlace" : "$items.itemId", "myStamp" : "$items.timestamp" } } } } ).pretty()
Running the above will produce this output:
{
"_id" : "Ursula",
"items" : [
{
"myPlace" : "52",
"myStamp" : ISODate("2016-02-13T18:07:32Z")
},
{
"myPlace" : "51",
"myStamp" : ISODate("2016-02-13T18:07:31Z")
},
{
"myPlace" : "50",
"myStamp" : ISODate("2016-02-13T18:07:30Z")
}
]
}
In MongoDB version 3.2.x, you can also use the $out operator in the very last stage of the aggregation pipeline, and have the output of the aggregation query written to a collection. Here is the code I used:
> db.bookmarks.aggregate(
{ "$match" : { "userId" : "Ursula" } },
{ "$unwind" : "$items" },
{ "$sort" : { "items.timestamp" : -1 } },
{ "$skip" : 2 },
{ "$limit" : 3 },
{ "$group" : { "_id" : "$userId", items: { "$push" : { "myPlace" : "$items.itemId", "myStamp" : "$items.timestamp" } } } },
{ "$out" : "ursula" } )
This gives me a collection named "ursula":
> show collections
ursula
and I can query that collection:
> db.ursula.find().pretty()
{
"_id" : "Ursula",
"items" : [
{
"myPlace" : "52",
"myStamp" : ISODate("2016-02-13T18:07:32Z")
},
{
"myPlace" : "51",
"myStamp" : ISODate("2016-02-13T18:07:31Z")
},
{
"myPlace" : "50",
"myStamp" : ISODate("2016-02-13T18:07:30Z")
}
]
}
>
Last of all, this is the input document I used in the aggregation query. You can compare this document to how I coded the aggregation query to see how I built the new items array.
> db.bookmarks.find( { "userId" : "Ursula" } ).pretty()
{
"_id" : ObjectId("56c240ed55f2f6004dc3b25c"),
"userId" : "Ursula",
"items" : [
{
"itemId" : "48",
"timestamp" : ISODate("2016-02-13T18:07:28Z")
},
{
"itemId" : "49",
"timestamp" : ISODate("2016-02-13T18:07:29Z")
},
{
"itemId" : "50",
"timestamp" : ISODate("2016-02-13T18:07:30Z")
},
{
"itemId" : "51",
"timestamp" : ISODate("2016-02-13T18:07:31Z")
},
{
"itemId" : "52",
"timestamp" : ISODate("2016-02-13T18:07:32Z")
},
{
"itemId" : "53",
"timestamp" : ISODate("2016-02-13T18:07:33Z")
},
{
"itemId" : "54",
"timestamp" : ISODate("2016-02-13T18:07:34Z")
}
]
}

mongodb aggregation match multiple $and on the same field

i have a document like this :
{
"ExtraFields" : [
{
"value" : "print",
"fieldID" : ObjectId("5535627631efa0843554b0ea")
},
{
"value" : "14",
"fieldID" : ObjectId("5535627631efa0843554b0eb")
},
{
"value" : "POLYE",
"fieldID" : ObjectId("5535627631efa0843554b0ec")
},
{
"value" : "30",
"fieldID" : ObjectId("5535627631efa0843554b0ed")
},
{
"value" : "0",
"fieldID" : ObjectId("5535627631efa0843554b0ee")
},
{
"value" : "0",
"fieldID" : ObjectId("5535627731efa0843554b0ef")
},
{
"value" : "0",
"fieldID" : ObjectId("5535627831efa0843554b0f0")
},
{
"value" : "42",
"fieldID" : ObjectId("5535627831efa0843554b0f1")
},
{
"value" : "30",
"fieldID" : ObjectId("5535627831efa0843554b0f2")
},
{
"value" : "14",
"fieldID" : ObjectId("5535627831efa0843554b0f3")
},
{
"value" : "19",
"fieldID" : ObjectId("5535627831efa0843554b0f4")
}
],
"id" : ObjectId("55369e60733e4914550832d0"), "title" : "A product"
}
what i want is to match one or more sets from the ExtraFields array. For example, all the products that contain the values print and 30. Since a value may be found in more than one fieldID (like 0 or true) we need to create a set like
WHERE (fieldID : ObjectId("5535627631efa0843554b0ea"), value : "print")
Where i'm having problems is when querying more than one fields. The pipeline i came up with is :
db.products.aggregate([
{'$unwind': '$ExtraFields'},
{
'$match': {
'$and': [{
'$and': [{'ExtraFields.value': {'$in': ["A52A2A"]}}, {
'ExtraFields.fieldID': ObjectId("5535627631efa0843554b0ea")
}]
}
,
{
'$and': [{'ExtraFields.value': '14'}, {'ExtraFields.fieldID': ObjectId("5535627631efa0843554b0eb")}]
}
]
}
},
]);
This returns zero results, but this is what i want to do in theory. Match all items that contain set 1 AND all that contain set 2.
The end result should look like a faceted search output :
[
{
"_id" : {
"values" : "18",
"fieldID" : ObjectId("5535627831efa0843554b0f3")
},
"count" : 2
},
{
"_id" : {
"values" : "33",
"fieldID" : ObjectId("5535627831efa0843554b0f2")
},
"count" : 1
}
]
Any ideas?
You could try the following aggregation pipeline
db.products.aggregate([
{
"$match": {
"ExtraFields.value": { "$in": ["A52A2A", "14"] },
"ExtraFields.fieldID": {
"$in": [
ObjectId("5535627631efa0843554b0ea"),
ObjectId("5535627631efa0843554b0eb")
]
}
}
},
{
"$unwind": "$ExtraFields"
},
{
"$match": {
"ExtraFields.value": { "$in": ["A52A2A", "14"] },
"ExtraFields.fieldID": {
"$in": [
ObjectId("5535627631efa0843554b0ea"),
ObjectId("5535627631efa0843554b0eb")
]
}
}
},
{
"$group": {
"_id": {
"value": "$ExtraFields.value",
"fieldID": "$ExtraFields.fieldID"
},
"count": {
"$sum": 1
}
}
}
])
With the sample document provided, this gives the output:
/* 1 */
{
"result" : [
{
"_id" : {
"value" : "14",
"fieldID" : ObjectId("5535627631efa0843554b0eb")
},
"count" : 1
}
],
"ok" : 1
}