Use $Redact with a regular expression - mongodb

In an aggregation pipeline, I am trying to filter some elements of an array of objects, based on the value of a field in this object.
Let's say that I have this entry:
{
"_id": "5b8911d346d19645f8a66bf4",
"title": "test task",
"creation_date": "2018-08-31T10:00:51.598Z",
"logs": [
{
"_id": "5b89126c46d19645f8a66bfb",
"content": "Running"
},
{
"_id": "5b89128646d19645f8a66bfd",
"content": "Stopping"
},
{
"_id": "5b89128646d19645f8a66bfd",
"content": "Stopped"
}
]
}
My objectif is to filter only the logs containing the stop word in their content:
{
"_id": "5b8911d346d19645f8a66bf4",
"title": "test task",
"creation_date": "2018-08-31T10:00:51.598Z",
"logs": [
{
"_id": "5b89128646d19645f8a66bfd",
"content": "Stopping"
},
{
"_id": "5b89128646d19645f8a66bfd",
"content": "Stopped"
}
]
}
I tried to use $redact to eliminate all the logs that does not contain the stop word:
$redact: {
$cond: {
if: { $match: { "logs.content": { $regex: "stop", $options: 'i' }}},
then: "$$KEEP",
else: "$$PRUNE"
}
}
but I keep getting the error:
Unrecognized expression '$match'

You can try below aggregation
db.collection.aggregate([
{ "$addFields": {
"logs": {
"$filter": {
"input": "$logs",
"cond": {
"$ne": [
{ "$indexOfBytes": [
{ "$toUpper": "$$this.content" },
{ "$toUpper": "stop" }
]},
-1
]
}
}
}
}}
])
Output
[
{
"_id": "5b8911d346d19645f8a66bf4",
"creation_date": "2018-08-31T10:00:51.598Z",
"logs": [
{
"_id": "5b89128646d19645f8a66bfd",
"content": "Stopping"
},
{
"_id": "5b89128646d19645f8a66bfd",
"content": "Stopped"
}
],
"title": "test task"
}
]

As per your requirement below query is working and it is properly tested
db.users.aggregate(
// Pipeline
[
// Stage 1
{
$unwind: {
path : "$logs",
preserveNullAndEmptyArrays : true // optional
}
},
// Stage 2
{
$group: {
_id: "$_id",
"title" :{$last:"$title"} ,
"creation_date" :{$last:"$creation_date"},
logs: {
$push: {
$cond: [ {$or:[{"$eq":[{ "$substr": [ "$logs.content", 0, 4 ] }, "Stop"]},{"$eq":[{ "$substr": [ "$logs.content", 0, 4 ] }, "stop"]}]},{"_id":"$logs._id","content":"$logs.content"},null]
}
}
}
},
// Stage 3
{
$project: {
logs: {
$filter: {
input: "$logs",
as: "log",
cond: { $ne: [ "$$log", null ] }
}
}
}
},
]
// Created with Studio 3T, the IDE for MongoDB - https://studio3t.com/
);

Related

MongoDB. Get every element from array in new field

I have a document with a nested array array_field:
{
"_id": {
"$oid": "1"
},
"id": "1",
"array_field": [
{
"data": [
{
"regions": [
{
"result": {
"item": [
"4",
"5",
"3"
]
}
},
{
"result": {
"item": [
"5"
]
}
},
{
"result": {
"item": [
"1"
]
}
}
]
}
]
}
]
}
I need add new field, new_added_field for example, with each array element from array_field.data.regions.result.item and remove array_field from document.
For example:
{
"_id": {
"$oid": "1"
},
"id": "1",
"new_added_field": [4,5,3,5,1]
}
I think i can do this with help of $unwind or $map but have difficulties and need dome hint, how i can do it with help op aggregation?
As you said,
db.collection.aggregate([
{
"$project": {
newField: {
"$map": {
"input": "$array_field",
"as": "m",
"in": "$$m.data.regions.result.item"
}
}
},
},
{ "$unwind": "$newField" },
{ "$unwind": "$newField" },
{ "$unwind": "$newField" },
{ "$unwind": "$newField" },
{
"$group": {
"_id": "$_id",
"newField": { "$push": "$newField" }
}
}
])
Working Mongo playground

Filter documents that have id in another collection in MongoDB with aggregation framework

So I have two collection. collectionA and collectionB
collection A has following documents
db={
"collectiona": [
{
"_id": "6173ddf33ed09368a094e68a",
"title": "a"
},
{
"_id": "61wefdf33ed09368a094e6dc",
"title": "b"
},
{
"_id": "61wefdfewf09368a094ezzz",
"title": "c"
},
],
"collectionb": [
{
"_id": "6173ddf33ed0wef368a094zq",
"collectionaID": "6173ddf33ed09368a094e68a",
"data": [
{
"userID": "123",
"visibility": false,
"response": false
},
{
"userID": "2345",
"visibility": true,
"response": true
}
]
},
{
"_id": "6173ddf33ed09368awef4e68g",
"collectionaID": "61wefdf33ed09368a094e6dc",
"data": [
{
"userID": "5678",
"visibility": false,
"response": false
},
{
"userID": "674",
"visibility": true,
"response": false
}
]
}
]
}
So What I need is documents from collection A which has response false in collection B
and document should be sorted by first the ones that have visibility false and then the ones that have visibility true
for eg. userID : 123 should get 3 documents
{
"_id": "6173ddf33ed09368a094e68a",
"title": "a"
},
{
"_id": "61wefdf33ed09368a094e6dc",
"title": "b"
},
{
"_id": "61wefdfewf09368a094ezzz",
"title": "c"
},
whereas userID 2345 should get two
{
"_id": "61wefdf33ed09368a094e6dc",
"title": "b"
},
{
"_id": "61wefdfewf09368a094ezzz",
"title": "c"
},
User 674 will receive 3 objects from collection A but second would be in the last as it has visibility true for that document
{
"_id": "6173ddf33ed09368a094e68a",
"title": "a"
},
{
"_id": "61wefdfewf09368a094ezzz",
"title": "c"
},
{
"_id": "61wefdf33ed09368a094e6dc",
"title": "b"
},
MongoDB Playground link : https://mongoplayground.net/p/3rLry0FPlw-
Really appreciate the help. Thanks
You can start from collectionA:
$lookup the collectionB for the record related to the user specified
filter out collectionB documents according to response
assign a helper sortrank field based on the visibility and whether collectionaID is a match
$sort according to sortrank
wrangle back to the raw collection A
db.collectiona.aggregate([
{
"$lookup": {
"from": "collectionb",
let: {
aid: "$_id"
},
"pipeline": [
{
$unwind: "$data"
},
{
$match: {
$expr: {
$and: [
{
$eq: [
"$data.userID",
"2345"
]
},
{
$eq: [
"$collectionaID",
"$$aid"
]
}
]
}
}
}
],
"as": "collB"
}
},
{
$match: {
"collB.data.response": {
$ne: true
}
}
},
{
"$unwind": {
path: "$collB",
preserveNullAndEmptyArrays: true
}
},
{
"$addFields": {
"sortrank": {
"$cond": {
"if": {
$eq: [
"$collB.data.visibility",
false
]
},
"then": 1,
"else": {
"$cond": {
"if": {
$eq: [
"$collB.collectionaID",
"$_id"
]
},
"then": 3,
"else": 2
}
}
}
}
}
},
{
$sort: {
sortrank: 1
}
},
{
$project: {
collB: false,
sortrank: false
}
}
])
Here is the Mongo playground for your reference.

MongoDB Aggregation - Lookup pipeline not returning any documents

I'm having hard time getting $lookup with a pipeline to work in MongoDB Compass.
I have the following collections:
Toys
Data
[
{
"_id": {
"$oid": "5d233c3bb173a546386c59bb"
},
"type": "multiple",
"tags": [
""
],
"searchFields": [
"Jungle Stampers - Two",
""
],
"items": [
{
"$oid": "5d233c3cb173a546386c59bd"
},
{
"$oid": "5d233c3cb173a546386c59be"
},
{
"$oid": "5d233c3cb173a546386c59bf"
},
{
"$oid": "5d233c3cb173a546386c59c0"
},
{
"$oid": "5d233c3cb173a546386c59c1"
},
{
"$oid": "5d233c3cb173a546386c59c2"
},
{
"$oid": "5d233c3cb173a546386c59c3"
},
{
"$oid": "5d233c3cb173a546386c59c4"
}
],
"name": "Jungle Stampers - Two",
"description": "",
"status": "active",
"category": {
"$oid": "5cfe727cac920000086b880e"
},
"subCategory": "Stamp Sets",
"make": "",
"defaultCharge": null,
"defaultOverdue": null,
"sizeCategory": {
"$oid": "5d0cfde57561e107c88fbde3"
},
"ageFrom": {
"$numberInt": "24"
},
"ageTo": {
"$numberInt": "120"
},
"images": [
{
"_id": {
"$oid": "5d233c3bb173a546386c59bc"
},
"id": {
"$oid": "5d233c39b173a546386c59ba"
},
"url": "/toyimages/5d233c39b173a546386c59ba.jpg",
"thumbUrl": "/toyimages/thumbs/tn_5d233c39b173a546386c59ba.jpg"
}
],
"__v": {
"$numberInt": "2"
}
}
]
Loans
Data
[
{
"_id": {
"$oid": "5e1f1661b712215978c746d9"
},
"tags": [],
"member": {
"$oid": "5e17495e4f81ab3f900dbb63"
},
"source": "admin portal - potter1#gmail.com",
"items": [
{
"id": {
"$oid": "5e1f160eb712215978c746d5"
},
"status": "new",
"_id": {
"$oid": "5e1f1661b712215978c746db"
},
"toy": {
"$oid": "5d233c3bb173a546386c59bb"
},
"cost": {
"$numberInt": "0"
}
},
{
"id": {
"$oid": "5e1f160eb712215978c746d5"
},
"status": "new",
"_id": {
"$oid": "5e1f1661b712215978c746da"
},
"toy": {
"$oid": "5d233b1ab173a546386c59b5"
},
"cost": {
"$numberInt": "0"
}
}
],
"dateEntered": {
"$date": {
"$numberLong": "1579095632870"
}
},
"dateDue": {
"$date": {
"$numberLong": "1579651200000"
}
},
"__v": {
"$numberInt": "0"
}
}
]
I am trying to return a list of toys and their associated loans that have a status of 'new' or 'out'.
I can use the following $lookup aggregate to fetch all loans:
{
from: 'loans',
localField: '_id',
foreignField: 'items.toy',
as: 'loansSimple'
}
However I am trying to use a pipeline to load loans that have the two statuses I am interested in, but it always only returns zero documents:
{
from: 'loans',
let: {
'toyid': '$_id'
},
pipeline: [
{
$match: {
$expr: {
$and: [
{$eq: ['$items.toy', '$$toyid']},
{$eq: ['$items.status', 'new']} // changed from $in to $eq for simplicity
]
}
}
}
],
as: 'loans'
}
This always seems to return 0 documents, however I arrange it:
Have I made a mistake somewhere?
I'm using MongoDB Atlas, v4.2.2, MongoDB Compass v 1.20.4
You are trying to search $$toyid inside inner array, but Operator Expression $eq cannot resolve it.
Best solution: $let (returns filtered loans by criteria) + $filter (applies filter for inner array) operator helps us to get desired result.
db.toys.aggregate([
{
$lookup: {
from: "loans",
let: {
"toyid": "$_id",
"toystatus": "new"
},
pipeline: [
{
$match: {
$expr: {
$gt: [
{
$size: {
$let: {
vars: {
item: {
$filter: {
input: "$items",
as: "tmp",
cond: {
$and: [
{
$eq: [
"$$tmp.toy",
"$$toyid"
]
},
{
$eq: [
"$$tmp.status",
"$$toystatus"
]
}
]
}
}
}
},
in: "$$item"
}
}
},
0
]
}
}
}
],
as: "loans"
}
}
])
MongoPlayground
Alternative solution 1. Use $unwind to flatten items attribute. (We create extra field named tmp which stores items value, flatten it with $unwind operator, match as you were doing and then exclude from result)
db.toys.aggregate([
{
$lookup: {
from: "loans",
let: {
"toyid": "$_id"
},
pipeline: [
{
$addFields: {
tmp: "$items"
}
},
{
$unwind: "$tmp"
},
{
$match: {
$expr: {
$and: [
{
$eq: [
"$tmp.toy",
"$$toyid"
]
},
{
$eq: [
"$tmp.status",
"new"
]
}
]
}
}
},
{
$project: {
tmp: 0
}
}
],
as: "loans"
}
}
])
MongoPlayground
Alternative solution 2. We use $reduce to create toy's array and with $in operator we check if toyid exists inside this array.
db.toys.aggregate([
{
$lookup: {
from: "loans",
let: {
"toyid": "$_id"
},
pipeline: [
{
$addFields: {
toys: {
$reduce: {
input: "$items",
initialValue: [],
in: {
$concatArrays: [
"$$value",
[
"$$this.toy"
]
]
}
}
}
}
},
{
$match: {
$expr: {
$in: [
"$$toyid",
"$toys"
]
}
}
},
{
$project: {
toys: 0
}
}
],
as: "loans"
}
}
])
$expr receives aggregation expressions, At that point $$items.toy is parsed for each element in an array as you would expect (however if it would it will still give you "bad" results as you'll get loans that have the required toy id and any other item with status new in their items array).
So you have two options to work around this:
If you don't care about the other items in the lookup'd document you can add an $unwind stage at the start of the lookup pipeline like so:
{
from: 'loans',
let: {
'toyid': '$_id'
},
pipeline: [
{
$unwind: "$items"
},
{
$match: {
$expr: {
$and: [
{$eq: ['$items.toy', '$$toyid']},
{$eq: ['$items.status', 'new']} // changed from $in to $eq for simplicity
]
}
}
}
],
as: 'loans'
}
If you do care about them just iterate the array in one of the possible ways to get a 'correct' match, here is an example using $filter
{
from: 'loads',
let: {
'toyid': '$_id'
},
pipeline: [
{
$addFields: {
temp: {
$filter: {
input: "$items",
as: "item",
cond: {
$and: [
{$eq: ["$$item.toy", "$$toyid"]},
{$eq: ["$$item.status", "new"]}
]
}
}
}
}
}, {$match: {"temp.0": {exists: true}}}
],
as: 'loans'
}

Is there a way in mongodb to group at multiple levels

I have a document which contains an array of array as given below.
This is the first document.
{
"_id": "5d932a2178fdfc4dc41d75da",
"data": [
{
"nestedData": [
{
"_id": "5d932a2178fdfc4dc41d75e1",
"name": "Special 1"
},
{
"_id": "5d932a2178fdfc4dc41d75e0",
"name": "Special 2"
}
]
}
]
}
I need to lookup(join) to another collection with the _id in the nestedData array in the aggregation framework.
The 2nd document from which I need to lookup is
{
"_id": "5d8b1ac3b15bc72d154408e1",
"status": "COMPLETED",
"rating": 4
}
I know I need to $unwind it twice to convert nestedData array into object.
But how do I group back again to form the same object like given below
{
"_id": "5d932a2178fdfc4dc41d75da",
"data": [
{
"array": [
{
"_id": "5d932a2178fdfc4dc41d75e1",
"name": "Special 1",
"data": {
"_id": "5d8b1ac3b15bc72d154408e1",
"status": "COMPLETED",
"rating": 4
},
{
"_id": "5d932a2178fdfc4dc41d75e0",
"name": "Special 2",
"data": {
"_id": "5d8b1ac3b15bc72d154408e0",
"status": "COMPLETED",
"rating": 4
},
}
]
}
]
}
Try this query
db.testers.aggregate([
{$lookup: {
from: 'demo2',
pipeline: [
{ $sort: {'_id': 1}},
],
as: 'pointValue',
}},
{
$addFields:{
"data":{
$map:{
"input":"$data",
"as":"doc",
"in":{
$mergeObjects:[
"$$doc",
{
"nestedData":{
$map:{
"input":"$$doc.nestedData",
"as":"nestedData",
"in":{
$mergeObjects:[
{ $arrayElemAt: [ {
"$map": {
"input": {
"$filter": {
"input": "$pointValue",
"as": "sn",
"cond": {
"$and": [
{ "$eq": [ "$$sn._id", "$$nestedData._id" ] },
]
}
}
},"as": "data",
"in": {
"name": "$$nestedData.name",
"data":"$$data",
}}
}, 0 ] },'$$nestedData'
],
}
}
}
}
]
}
}
}
}
},
{$project: { pointValue: 0 } }
]).pretty()

Aggregate Unexpected token :

Below aggregate mongodb query gives
Unexpected token : error
db.getCollection("products_data").aggregate(
{
"$unwind": {
"path": "$color",
"preserveNullAndEmptyArrays": true
}
},
{
"$match":{
"country":"UK",
"$or":[{
"$and":[
"$or":
[{
"$and":[
{"status":"drafted"},
{"color":{$in:["blue"]}}
]},
{"$and":[
{"status1":"complete"},
{"status2":{$nin:["n/a","drafted","complete"]}},
{"color":{$in:["green"]}}
]}
]
]
},{
"$and":[
"$or":
[
{ "$and":[
{"status":"drafted"},
{"color":{$in:["blue"]}}
]},
{"$and":[
{"status1":"complete"},
{"status2":{$nin:["n/a","drafted","complete"]}},
{"color":{$in:["green"]}}
]}
]
]
}
]
}
},
{
"$group":{
"_id":"$field",
"products":{$sum: 1},
"bid":{"$push":"$product_id"}
}
},
{
"$project":{
"field":"$_id",
"products":"$products",
"bid":1,
"_id":0
}
}
);
To fetch the aggregate count for the given specified condition.
Correct syntax to use aggregate and it's stages in pipeline
db.getCollection("products_data").aggregate([
{ "$unwind": { "path": "$color", "preserveNullAndEmptyArrays": true }},
{ "$match": {
"country": "UK",
"$or": [
{
"$and": [
{
"$or": [
{ "$and": [{ "status": "drafted" }, { "color": { "$in": ["blue"] }}] },
{ "$and": [{ "status1": "complete" }, { "status2": { "$nin": ["n/a", "drafted", "complete"] }}, { "color": { "$in": ["green"] }}]}
]
}
]
},
{
"$and": [
{
"$or": [
{ "$and": [{ "status": "drafted" }, { "color": { "$in": ["blue"] }}] },
{ "$and": [{ "status1": "complete" }, { "status2": { "$nin": ["n/a", "drafted", "complete"] }}, { "color": { "$in": ["green"] }}] }
]
}
]
}
]
}},
{ "$group": {
"_id": "$field",
"products": { "$sum": 1 },
"bid": { "$push": "$product_id" }
}},
{ "$project": { "field": "$_id", "products": "$products", "bid": 1, "_id": 0 }}
])