How to save deletion in a deeply nested MongoDB document - mongodb

I am new to MongoDB and I am using MongoDB shell to perform the operations.
I am working to remove the array named Process from all the Items, but it seems that I do not grasp the remove concept correctly.
The documents we use are deeply nested - we do not know how many items there are, or how deep the level of nesting.
What I tried so far is to use recursion to iterate through the items:
function removeAllProcessFields(docItems)
{
if(Array.isArray(docItems))
{
docItems.forEach(function(item)
{
print("idItem: "+item._id);
if(item.Process == null)
{
print("Process null");
}
else
{
$unset: { Process: ""}
}
removeAllProcessFields(item.Items);
})
}
}
var docs = db.getCollection('MyCollection').find({})
docs.forEach(function(doc)
{
print("idDoc: "+doc._id);
removeAllProcessFields(doc.Items);
})
But I have difficulties on using unset properly to save the operation.
An example document would be:
{
"_id": "622226d319517e83e8ed6151",
"Name": "test1",
"Description": "",
"Items": [{
"_id": "622226d319517e83e8ed614e",
"Name": "test-item",
"Description": "",
"Process": [{
"Name": "Step1"
}, {
"Name": "Step2"
}],
"Items": [{
"_id": "622226d319517e83e8ed614f",
"Name": "test-subItem1",
"Description": "",
"Process": [{
"Name": "StepSub1"
}, {
"Name": "StepSub2"
}, {
"Name": "StepSub3"
}],
"Items": []
},
{
"_id": "622226d319517e83e8ed6150",
"Name": "test-subItem2",
"Description": "",
"Process": [{
"Name": "StepSub4"
}, {
"Name": "StepSub5"
}, {
"Name": "StepSub6"
}],
"Items": []
}
]
}]
}
What I hope to achieve would be:
{
"_id": "622226d319517e83e8ed6151",
"Name": "test1",
"Description": "",
"Items": [{
"_id": "622226d319517e83e8ed614e",
"Name": "test-item",
"Description": "",
"Items": [{
"_id": "622226d319517e83e8ed614f",
"Name": "test-subItem1",
"Description": "",
"Items": []
},
{
"_id": "622226d319517e83e8ed6150",
"Name": "test-subItem2",
"Description": "",
"Items": []
}
]
}]
}

Something like this maybe using the $[] positional operator:
db.collection.update({},
{
$unset: {
"Items.$[].Items.$[].Process": 1,
"Items.$[].Process": 1
}
})
You just need to construct it in the recursion ...
playground
JavaScript recursive function example:
mongos> db.rec.find()
{ "_id" : ObjectId("622a6c46ae295edb276df8e2"), "Items" : [ { "a" : 1 }, { "Items" : [ { "Items" : [ { "Items" : [ ], "Process" : [ 1, 2, 3 ] } ], "Process" : [ 4, 5, 6 ] } ], "Process" : [ ] } ] }
mongos> db.rec.find().forEach(function(obj){ var id=obj._id,ar=[],z=""; function x(obj){ if(typeof obj.Items != "undefined" ){ obj.Items.forEach(function(k){ if( typeof k.Process !="undefined" ){ z=z+".Items.$[]";ar.push(z.substring(1)+".Process") }; if(typeof k.Items != "undefined"){x(k)}else{} }) }else{} };x(obj);ar.forEach(function(del){print( "db.collection.update({_id:ObjectId('"+id+"')},{$unset:{'"+del+"':1}})" );}) })
db.collection.update({_id:ObjectId('622a6c46ae295edb276df8e2')},{$unset:{'Items.$[].Process':1}})
db.collection.update({_id:ObjectId('622a6c46ae295edb276df8e2')},{$unset:{'Items.$[].Items.$[].Process':1}})
db.collection.update({_id:ObjectId('622a6c46ae295edb276df8e2')},{$unset:{'Items.$[].Items.$[].Items.$[].Process':1}})
mongos>
Explained:
Loop over all documents in collection with forEach
Define recursive function x that will loop over any number of nested Items and identify if there is Process field and push to array ar
Finally loop over array ar and construct the update $unset query , in the example only printed for safety , but you can improve generating single query per document and executing unset query ...

Assuming you are on v>=4.4 you can use the "merge onto self" feature of $merge plus defining a recursive function to sweep through the collection and surgically remove one or a list of fields at any level of the hierarchy. The same sort of needs arise when processing json-schema data which is also arbitrarily hierarchical.
The solution below has extra logic to "mark" documents that had any modifications so the others can be removed from the update set passed to $merge. It also can be further refined to reduce some variables; it was edited down from a more general solution that had to examine keys and values.
db.foo.aggregate([
{$replaceRoot: {newRoot: {$function: {
body: function(obj, target) {
var didSomething = false;
var process = function(holder, spot, value) {
// test FIRST since [] instanceof Object is true!
if(Array.isArray(value)) {
for(var jj = value.length - 1; jj >= 0; jj--) {
process(value, jj, value[jj]);
}
} else if(value instanceof Object) {
walkObj(value);
}
};
var walkObj = function(obj) {
Object.keys(obj).forEach(function(k) {
if(target.indexOf(k) > -1) {
delete obj[k];
didSomething = true;
} else {
process(obj, k, obj[k]);
}
});
}
// ENTRY POINT:
if(!Array.isArray(target)) {
target = [ target ]; // if not array, make it an array
}
walkObj(obj);
if(!didSomething) {
obj['__didNothing'] = true;
}
return obj;
},
// Invoke!
// You can delete multiple fields with an array, e.g.:
// ..., ['Process','Description']
args: [ "$$ROOT", 'Process' ],
lang: "js"
}}
}}
// Only let thru docs WITHOUT the marker:
,{$match: {'__didNothing':{$exists:false}} }
,{$merge: {
into: "foo",
on: [ "_id" ],
whenMatched: "merge",
whenNotMatched: "fail"
}}
]);

Related

How can I filter records with multiple $in condition (some are optional $in) with arrays of string? Mongoose/MongoDB

You can see my Mongodb Records at last... I am now trying to implement search functionality,
I mad checkbox filtration for my project and below I listed those arrays after I clicked multiple checkboxes (see 1, 2 and 3).
I tried in aggregate with multiple match queries with $in, but it doesn't worked. Below arrays are used to check the records.
for example:
["Restaurant", "Mall"] need to check with "commercialType" in records, at the same time ["AC Rooms", "3 Phase Electricity"] need to check with "propertyFeatures.name" in records.. so all matching records must display if records exist with those filtrations.
I tried with multiple $in queries like this, but it gives empty records.
"$match": {
"commercialType": {
"$in": ["Restaurant", "Hotel"]
},
{
"propertyFeatures.name": {
"$in": ['AC Rooms']
}
},
... other match filters
}
1. Below Array is used to find commercialType (field in doc)
[
'Restaurant',
'Office space',
'Hotel'
]
2. Below Array is used to find landType (field in doc)
[
'Bare land',
'Beachfront land',
'Coconut land'
]
3. Below Array is used to find "propertyFeatures.name" (field in doc)
[
'AC Rooms',
'3 Phase Electricity',
'Hot Water'
]
[
{
"_id": {
"$oid": "6343b68edf5e889a575c8502"
},
"propertyType": "House",
"propertyFeatures": [
{
"id": 1,
"name": "AC Rooms",
"value": true
}
]
},
{
"_id": {
"$oid": "6343b68edf5e889a575c8502"
},
"propertyType": "Land",
"landType": "Bare land",
"propertyFeatures": [
{
"id": 1,
"name": "Wider Road",
"value": true
}
]
},
{
"_id": {
"$oid": "6343b68edf5e889a575c8502"
},
"propertyType": "Commercial",
"commercialType": "Restaurant",
"propertyFeatures": [
{
"id": 1,
"name": "3 Phase Electricity",
"value": true
}
]
}
]
You are probably missing $or operator, so your example pipeline becomes
[
{"$match": {
"$or": [
{
"commercialType": {
"$in": ["Restaurant", "Hotel"]
},
{
"propertyFeatures.name": {
"$in": ['AC Rooms']
}
}
]
}
]
MongoDB docs: https://www.mongodb.com/docs/manual/reference/operator/aggregation/or/#error-handling

Sanity.io GROQ query for array of objects

I'm learning to code and now I am on the stage of a small pet project with Sanity as a CMS.
Long story short, making an API I'm trying to fetch cocktails data with votes for the cocktails. The votes are stored within persons who voted:
GROQ query
*[
_type == "cocktail" &&
!(_id in path('drafts.**'))
] {
name,
_id,
"votes" : *[_type == "person" && references(^._id)] {
votes[] {
score,
"id": cocktail._ref
}
}
}
which returns
[
{
"_id": "pdUGiuRzgLGpnc4cfx76nA",
"name": "Cuba Libre",
"votes": [
{
"votes": {
"id": "pdUGiuRzgLGpnc4cfx76nA",
"score": 2
}
},
{
"votes": {
"id": "pdUGiuRzgLGpnc4cfx76nA",
"score": 2
}
}
]
},
{
"_id": "pdUGiuRzgLGpnc4cfxBOyM",
"name": "The ERSH7",
"votes": []
}
]
As you can see, the merge provides embedded arrays of votes meanwhile I want sth like:
[{
...cocktail attributes...
"votes" : [
{score: 2, id: pdUGiuRzgLGpnc4cfx76nA},
{score: 2, id: pdUGiuRzgLGpnc4cfx76nA}
]
}
... more cocktails....
]
Trying to get this I modified the query:
*[
_type == "cocktail" &&
!(_id in path('drafts.**'))
] {
name,
_id,
"votes" : *[_type == "person" && references(^._id)].votes[] {
score,
"id": cocktail._ref
}
}
which should take a projection from every element of the votes arr. Unfortunately I get empty arrays:
[
{
"_id": "pdUGiuRzgLGpnc4cfx76nA",
"name": "Cuba Libre",
"votes": [
{},
{}
]
}
...more cocktails
]
How can I achieve the desired result?
Thank you for reading! Would appreciate any help!
Yes, had similar struggles with "flattening" the projections my self. I solved it with dot-syntax. Try just adding .votes on your first attempt:
*[
_type == "cocktail" &&
!(_id in path('drafts.**'))
]
{
name,
_id,
"votes" : *[_type == "person" && references(^._id)] {
votes[] {
score,
"id": cocktail._ref
}
}
.votes
}
If this is correct, the whole query can be simplified but I'm not at the level, yet, where I can do that without testing against a similar set ,'-)
Actually, GROQ has syntax to flatten array starting from v2021-03-25
find this example:
{
'nestedArray': [
{'foo': [1,2,3,4,5]},
{'foo': [6,7,8,9,10,11,12]},
{'foo': [13,14,15]},
]
}{
'stillNestedArray': #.nestedArray[].foo,
'flatArray': #.nestedArray[].foo[]
}
Note the [] right after foo - this is what's flatten it
So the original query from the question should look like
*[
_type == "cocktail" &&
!(_id in path('drafts.**'))
] {
name,
_id,
"votes" : *[_type == "person" && references(^._id)] {
votes[] {
score,
"id": cocktail._ref
}
}.votes[]
}

MongoDb aggregation project onto collection

I've a problem with a huge MongoDb aggregation pipeline. I've many constraint and I've simplified the problem a lot. Hence, don't discuss the goal for this query.
I've a mongo aggregation that gives something similar to this:
[
{
"content": {
"processes": [
{
"id": "101a",
"title": "delivery"
},
{
"id": "101b",
"title": "feedback"
}
]
}
}
]
To this intermediate result I'm forced to apply a project operation in order to obtain something similar to this:
[
{
"results":
{
"titles": [
{
"id": "101a",
"value": "delivery"
},
{
"id": "101b",
"value": "feedback"
}
]
}
}
]
enter code here
But applying this projections:
"results.titles.id": "$content.processes.id",
"results.titles.value": "$content.processes.title"
I obtain this:
[
{
"results":
{
"titles": {
"id": ["101a", "101b"]
"value": ["delivery", "feedback"]
}
}
}
}
]
Collection are created but not in the proper position.
Is it possible to exploit some operator inside the project operation in order to tell mongo to create an array in a parent position?
Something like this:
"results.titles.$[x].value" : "$content.processes.value"
You can use the dot notation to project entire array:
db.col.aggregate([
{
$project: {
"results.titles": "$content.processes"
}
}
])
and if you need to rename title to value then you have to apply $map operator:
db.col.aggregate([
{
$project: {
"results.titles": {
$map: {
input: "$content.processes",
as: "process",
in: {
id: "$$process.id",
value: "$$process.title"
}
}
}
}
}
])

How to project selected fields from a document in mongoose based on conditions

I have my document structure in this way. Where i am storing all the events based occured with timestamps. My document look like this
[
{
"_id": {
"$oid": "589341cff92be305c034cb5a"
},
"__v": 0,
"name": "TV",
"switch_event": [
{
"timestamp": 1486186277826,
"event_type": "on"
},
{
"timestamp": 1486272677826,
"event_type": "off"
},
{
"timestamp": 1486099877826,
"event_type": "off"
},
{
"timestamp": 1486186277826,
"event_type": "on"
},
{
"timestamp": 1486272677826,
"event_type": "off"
},
{
"timestamp": 1486099877826,
"event_type": "off"
}
]
}
]
Now while querying for this document i am interested in only the events which occured today. So after querying for this i am writing projection query like this (for testing i kept timestamp > 0, which should give all events) -
SwitchAppliance.find({_id:"589341cff92be305c034cb5a"},{
name:1,
switch_event:{$elemMatch: {
timestamp: {
$gt:0
}
}}
},(err,results)=>{
if(err) {console.log(err);return next({message: "Internal Server Error"});}
return res.json(results);
} );
But when i am getting result i am only getting one event object in the switch_event array- like this -
[
{
"_id": "589341cff92be305c034cb5a",
"switch_event": [
{
"_id": "589567251c653a0890b8b1ef",
"event_type": "on",
"timestamp": 1486186277826
}
],
"name": "TV"
}
]
You are querying Date.now() (timestamp or number) against new Date() (date object). A quick console test shows the following:
> typeof Date.now()
=> "number"
> typeof new Date()
=> "object"
So first off, I would change
var today = new Date();
to
var today = Date.now();
Then you are missing that "timestamp" does not hold a date value, but another Object with one Item with the key $date and a date as a value. Therefore your query should probably look similar to something like this:
var today = Date.now();
Item1.find({}).populate({
path:"item1",
select: {
name:1,
event:{
$elemMatch: {
timestamp: {
date: {
$gte:today
}
}
}
}
}
})
Finally, found the key here.
Unwind empty array in mongodb
here #ivan.srb 's answers came handy.
Finally ended up doing something like this -
SwitchAppliance.aggregate([
{$match:{_id:{$in : switch_appliances_array}}},
{$project:{
switch_no:1,switch_name:1,switch_type:1,appliance_type:1,sboard_id:1,current_status:1,
switch_event : { $cond : [ { $eq : [ "$switch_event", [] ] }, [ { timestamp : 0 , event_type:"off"} ], '$switch_event' ] }
}},
{$unwind:"$switch_event"},
{$match:{$or: [{"switch_event.timestamp":{$gt:now}},{ "switch_event.timestamp":0} ]}},
{$group:{
_id:"$_id",
switch_no:{$first:"$switch_no"},
switch_type:{$first:"$switch_type"},
switch_name:{$first:"$switch_name"},
appliance_type:{$first:"$appliance_type"},
current_status:{$first:"$current_status"},
switch_events:{$push:"$switch_event"}
}},

Add existing fields to nested schema

I have documents in my db with schema:
var MySchema = new Schema({
Street: { type: String },
Age: { type: Number, default: null },
Date: { type: Date },
Stuff: [
{
_id:false,
ThisDate: { type: Date },
ThisStreet: { type: String }
}]
});
Right now it is (Stuff is empty):
db.person.findOne()
{
Street: 'TheStreet',
Age: 23,
Date: ISODate("2016-02-19T00:00:00.000Z"),
Stuff: []
}
I then want to update all documents. What I want to do is to move Street and Date fields into Stuff array and delete Street and Date fields from schema.
Like this:
db.person.findOne()
{
Age: 23,
Stuff : [
{
ThisDate : ISODate("2016-02-19T00:00:00.000Z"),
ThisStreet : "TheStreet"
}
]
}
How could I achieve this?
Best Regards
Since this is a "one off" operation I would do it in the shell rather than use any other framework.
For MongoDB 3.2.x releases and greater use, bulkWrite():
var ops = [];
db.person.find({
"Street": { "$exists": true },
"Date": { "$exists": true }
}).forEach(function(doc) {
ops.push({
"updateOne": {
"filter": { "_id": doc._id },
"update": {
"$unset": {
"Street": "",
"Date": ""
},
"$set": {
"Stuff": [{
"ThisDate": doc.Date,
"ThisStreet": doc.Street
}]
}
}
}
});
if ( ops.length == 1000 ) {
db.person.bulkWrite(ops);
ops = [];
}
})
if ( ops.length > 0 )
db.person.bulkWrite(ops);
Or for MongoDB 2.6.x and 3.0.x releases use this version of Bulk operations:
var bulk = db.person.initializeUnorderedBulkOp(),
count = 0;
db.person.find({
"Street": { "$exists": true },
"Date": { "$exists": true }
}).forEach(function(doc) {
bulk.find({ "_id": doc._id }).updateOne({
"$unset": {
"Street": "",
"Date": ""
},
"$set": {
"Stuff": [{
"ThisDate": doc.Date,
"ThisStreet": doc.Street
}]
}
});
if ( count % 1000 == 0 ) {
bulk.execute();
bulk = db.person.initializeUnorderedBulkOp();
}
});
if ( count % 1000 != 0 )
bulk.execute();
Bottom line is that you need to iterate the documents in the collection and write them back with the re-arranged content "one by one". At least the Bulk operations API in use in both cases will reduce the load of writing and responding with the server to only one in every 1000 documents in the collection to process.
Also, rather than rewriting the whole document you are using $unset to remove the desired fields and $set to write "just" the data you want
Working example
db.person.insert(
{
"Street": 'TheStreet',
"Age": 23,
"Date": ISODate("2016-02-19T00:00:00.000Z"),
"Stuff": []
}
)
Then after running either pdate above the result is:
{
"_id" : ObjectId("56e607c1ca8e7e3519b4ce93"),
"Age" : 23,
"Stuff" : [
{
"ThisDate" : ISODate("2016-02-19T00:00:00Z"),
"ThisStreet" : "TheStreet"
}
]
}
I'd suggest you to transform document using aggregation framework and update as described in code snippet below
db.person.aggregate([
{$project:{Age:1, Stuff:[{Date:"$Date", Street:"$Street"}]}}
]).forEach(function(o){
var id = o._id;
delete o._id;
db.person.update({_id:id, Street:{$exists: true}},o);
});
After successful execution, you document or documents should look like
{
"_id" : ObjectId("56e2cd45792861e14df1f0a9"),
"Age" : 23.0,
"Stuff" : [
{
"Date" : ISODate("2016-02-19T00:00:00.000+0000"),
"Street" : "TheStreet"
}
]
}