Working around 16M limit with a join-like query - mongodb

Given the architecture below, is there a way that I can query the people collection based on the searchable columns and get back an object such as this?
organization with people: {
_id: "13efdsf31r23rwes"
name: "Name",
wp_slug: "/wordpress-slug/",
type: "Cemetary",
file_name: "something.csv",
columns: [
{ label: "Maiden Name", name: "maiden_name", hidden: false },
{ label: "First Name", name: "first_name", hidden: false },
{ label: "First Pet", name: "first_pet", hidden: true },
],
people: [
{
_id: "13okekdsf12313",
"organization_id": 13efdsf31r23rwes,
"maiden_name": "Gilomen",
"surname": "Black",
"first_name": "Maria",
"first_pet": "Bud the Dog",
"searchable_columns": {
"surname": ["Black", "Gilomen"],
"given": ["Maria"],
}
},
{
_id: "1231sdf1313ewfd",
"organization_id": 13efdsf31r23rwes,
"maiden_name": "Portland",
"surname": "Smith",
"first_name": "Rose",
"first_pet": "Max the Dog",
"searchable_columns": {
"surname": ["Smith", "Portland"],
"given": ["Rose"],
}
},
]
}
Current architecture:
organization document example: {
_id: "13efdsf31r23rwes"
name: "Name",
wp_slug: "/wordpress-slug/",
type: "Cemetary",
file_name: "something.csv",
columns: [
{ label: "Maiden Name", name: "maiden_name", hidden: false },
{ label: "First Name", name: "first_name", hidden: false },
{ label: "First Pet", name: "first_pet", hidden: true },
],
}
people collection: [
{
_id: "13okekdsf12313",
"organization_id": 13efdsf31r23rwes,
"maiden_name": "Gilomen",
"surname": "Black",
"first_name": "Maria",
"first_pet": "Bud the Dog",
"searchable_columns": {
"surname": ["Black", "Gilomen"],
"given": ["Maria"],
}
},
{
_id: "1231sdf1313ewfd",
"organization_id": 13efdsf31r23rwes,
"maiden_name": "Portland",
"surname": "Smith",
"first_name": "Rose",
"first_pet": "Max the Dog",
"searchable_columns": {
"surname": ["Smith", "Portland"],
"given": ["Rose"],
}
},
]
Unfortunately we have hit the 16M document limit when they are stored as a combined document, so I need to do this workaround. Is this possible to do with a straight MongoDB query, or is there no other way than to query twice? There are about 5 million records of people and 500+ organizations. Any help would be appreciated, especially if shown in MongoPHP format.

I was able to workaround the size limitation. Using the "desired structure", I split the intended document into multiple documents of a size under 16M. Using an aggregate query, I was able to combine all of the splits back together.
$documents_with_results = $db->organization_data->aggregate(array(
array( '$match' => $search_column_query),
array( '$unwind' => '$p' ),
array( '$match' => $search_column_query),
array( '$group' => array(
'_id' => '$_id',
'name' => array( '$first' => '$n' ),
'slug' => array( '$first' => '$ws' ),
'people' => array( '$push' => '$p' )
))
));

Related

mongo push to array in aggregation

I have the following code:
const newArray = [];
companies.items.forEach(async (item) => {
if (item.parentCompanyID) {
newArray.push({
updateOne: {
filter: { id: item?.parentCompanyID },
update: [
{
$push: {
branches: {
id: item?.id,
name: item.companyName,
parentId: item?.parentCompanyID,
type: item.companyType,
active: item.isActive,
number: item.companyNumber,
newlyAdded: { $eq: [{ $type: '$newlyAdded' }, 'missing'] },
},
},
},
],
upsert: true,
},
});
} else {
newArray.push({
updateOne: {
filter: { id: item?.id },
update: [
{
$set: {
id: item?.id,
name: item.companyName,
parentId: item?.parentCompanyID,
type: item.companyType,
active: item.isActive,
number: item.companyNumber,
newlyAdded: { $eq: [{ $type: '$newlyAdded' }, 'missing'] },
},
},
],
upsert: true,
},
});
}
});
await Company.bulkWrite(newArray);
This will go through company.items and for each will add updateOne into newArray which will the goes to bulkWrite.
My problem lies with $push as this needs to be in aggregation pipeline, and when i add the brackets around update it will break with MongooseError: Invalid update pipeline operator: "$push"
Iam sure the script could be simplified but iam still fairly new to mongoDB. What i need is this to insert to Company if the item hasnt got parentCompanyID, if it does have than push to branches array for the relevant Company with id of parentCompanyID.
Sample data from company.items array:
{
id: 5,
name: "Sports"
parentCompanyID: null
},
{
id: 51,
name: "Football"
parentCompanyID: 5
}
And MongoDB for COmpany should look like this:
{
id: 5,
name: "Sports",
parentCompanyID: null,
branches: [{
id: 51,
name: "Football",
parentCompanyID: 5
}]
}
Hope this makes sense. ANy help would be appreciated. I could not find any similar issue and only one i came accross was to use $concatArrays but this wouldnt work either.
Thank you
EDIT:
as per #Takis_ answer thsi now sort of works. Only problem is when $concatArrays does it jobs its not pushing into array as expected from $push. this is the result as of now, insted of branches being one array it has nested arrays. if there are more branches it follows same patter and it could end up with many nested arrays rather than 1 array of objects. any ideas?
{
"id": 29683585,
"name": "123",
"parentId": null,
"newlyAdded": true,
"branches": [
[
null,
{
"id": 29693873,
"name": "245",
"parentId": 29683585
}
],
{
"id": 29695646,
"name": "789",
"parentId": 29683585
}
]
}
This has now been sorted. Thanks to Takis and his pointing to $concatArrays i was able to make this works.
Working code is below:
newArray.push({
updateOne: {
filter: { id: item?.parentCompanyID },
update: [
{
$set: {
branches: {
$concatArrays: [
{ $ifNull: ['$branches', []] },
[
{
id: item?.id,
name: item.companyName,
parentId: item?.parentCompanyID,
type: item.companyType,
active: item.isActive,
number: item.companyNumber,
newlyAdded: { $ne: ['$newlyAdded', null] },
},
],
],
},
},
},
],
upsert: true,
},
});

Google Home Fan Speed probleme

I have the following answer from sync :
When i use the google home mini or simulator and say set the fan to low, I receive that device doesnt support that. Can anybody help me ? Documentation seems the problem
{
id: '123',
type: 'action.devices.types.FAN',
traits: [
'action.devices.traits.FanSpeed',
'action.devices.traits.OnOff'
],
name: {
defaultNames: ['Sirius Cybernetics Corporation 33321'],
name: 'Fan',
nicknames: ['wall fan']
},
willReportState: true,
attributes: {
availableFanSpeeds: {
speeds: [{
speed_name: 'Low',
speed_values: [{
speed_synonym: ['low', 'slow'],
lang: 'en'
}, {
speed_synonym: ['low', 'slow'],
lang: 'de'
}]
}, {
speed_name: 'High',
speed_values: [{
speed_synonym: ['high'],
lang: 'en'
}, {
speed_synonym: ['high'],
lang: 'de'
}]
}],
},
reversible: true
},
deviceInfo: {
manufacturer: 'Sirius Cybernetics Corporation',
model: '492134',
hwVersion: '3.2',
swVersion: '11.4'
},
}
EDIT:
I forgot to say that on off work well..
You are missing the attribute 'ordered' See below
attributes: {
availableFanSpeeds: {
speeds: [{
speed_name: 'Low',
speed_values: [{
speed_synonym: ['low', 'slow'],
lang: 'en'
}, {
speed_synonym: ['low', 'slow'],
lang: 'de'
}]
}, {
speed_name: 'High',
speed_values: [{
speed_synonym: ['high'],
lang: 'en'
}, {
speed_synonym: ['high'],
lang: 'de'
}]
}],
'ordered':true
},
reversible: true
}

Multiple array elements are not updating in mongodb [duplicate]

I need to modify a document inside an array that is inside another array.
I know MongoDB doesn't support multiple '$' to iterate on multiple arrays at the same time, but they introduced arrayFilters for that.
See: https://jira.mongodb.org/browse/SERVER-831
MongoDB's sample code:
db.coll.update({}, {$set: {“a.$[i].c.$[j].d”: 2}}, {arrayFilters: [{“i.b”: 0}, {“j.d”: 0}]})
Input: {a: [{b: 0, c: [{d: 0}, {d: 1}]}, {b: 1, c: [{d: 0}, {d: 1}]}]}
Output: {a: [{b: 0, c: [{d: 2}, {d: 1}]}, {b: 1, c: [{d: 0}, {d: 1}]}]}
Here's how the documents are set:
{
"_id" : ObjectId("5a05a8b7e0ce3444f8ec5bd7"),
"name" : "support",
"contactTypes" : {
"nonWorkingHours" : [],
"workingHours" : []
},
"workingDays" : [],
"people" : [
{
"enabled" : true,
"level" : "1",
"name" : "Someone",
"_id" : ObjectId("5a05a8c3e0ce3444f8ec5bd8"),
"contacts" : [
{
"_id" : ObjectId("5a05a8dee0ce3444f8ec5bda"),
"retries" : "1",
"priority" : "1",
"type" : "email",
"data" : "some.email#email.com"
}
]
}
],
"__v" : 0
}
Here's the schema:
const ContactSchema = new Schema({
data: String,
type: String,
priority: String,
retries: String
});
const PersonSchema = new Schema({
name: String,
level: String,
priority: String,
enabled: Boolean,
contacts: [ContactSchema]
});
const GroupSchema = new Schema({
name: String,
people: [PersonSchema],
workingHours: { start: String, end: String },
workingDays: [Number],
contactTypes: { workingHours: [String], nonWorkingHours: [String] }
});
I need to update a contact. This is what I tried using arrayFilters:
Group.update(
{},
{'$set': {'people.$[i].contacts.$[j].data': 'new data'}},
{arrayFilters: [
{'i._id': mongoose.Types.ObjectId(req.params.personId)},
{'j._id': mongoose.Types.ObjectId(req.params.contactId)}]},
function(err, doc) {
if (err) {
res.status(500).send(err);
}
res.send(doc);
}
);
The document is never updated and I get this response:
{
"ok": 0,
"n": 0,
"nModified": 0
}
What am I doing wrong?
So the arrayFilters option with positional filtered $[<identifier>] does actually work properly with the development release series since MongoDB 3.5.12 and also in the current release candidates for the MongoDB 3.6 series, where this will actually be officially released. The only problem is of course is that the "drivers" in use have not actually caught up to this yet.
Re-iterating the same content I have already placed on Updating a Nested Array with MongoDB:
NOTE Somewhat ironically, since this is specified in the "options" argument for .update() and like methods, the syntax is generally compatible with all recent release driver versions.
However this is not true of the mongo shell, since the way the method is implemented there ( "ironically for backward compatibility" ) the arrayFilters argument is not recognized and removed by an internal method that parses the options in order to deliver "backward compatibility" with prior MongoDB server versions and a "legacy" .update() API call syntax.
So if you want to use the command in the mongo shell or other "shell based" products ( notably Robo 3T ) you need a latest version from either the development branch or production release as of 3.6 or greater.
All this means is that the current "driver" implementation of .update() actually "removes" the necessary arguments with the definition of arrayFilters. For NodeJS this will be addressed in the 3.x release series of the driver, and of course "mongoose" will then likely take some time after that release to implement it's own dependencies on the updated driver, which would then no longer "strip" such actions.
You can however still run this on a supported server instance, by dropping back to the basic "update command" syntax usage, since this bypassed the implemented driver method:
const mongoose = require('mongoose'),
Schema = mongoose.Schema,
ObjectId = mongoose.Types.ObjectId;
mongoose.Promise = global.Promise;
mongoose.set('debug',true);
const uri = 'mongodb://localhost/test',
options = { useMongoClient: true };
const contactSchema = new Schema({
data: String,
type: String,
priority: String,
retries: String
});
const personSchema = new Schema({
name: String,
level: String,
priority: String,
enabled: Boolean,
contacts: [contactSchema]
});
const groupSchema = new Schema({
name: String,
people: [personSchema],
workingHours: { start: String, end: String },
workingDays: { type: [Number], default: undefined },
contactTypes: {
workingHours: { type: [String], default: undefined },
contactTypes: { type: [String], default: undefined }
}
});
const Group = mongoose.model('Group', groupSchema);
function log(data) {
console.log(JSON.stringify(data, undefined, 2))
}
(async function() {
try {
const conn = await mongoose.connect(uri,options);
// Clean data
await Promise.all(
Object.entries(conn.models).map(([k,m]) => m.remove() )
);
// Create sample
await Group.create({
name: "support",
people: [
{
"_id": ObjectId("5a05a8c3e0ce3444f8ec5bd8"),
"enabled": true,
"level": "1",
"name": "Someone",
"contacts": [
{
"type": "email",
"data": "adifferent.email#example.com"
},
{
"_id": ObjectId("5a05a8dee0ce3444f8ec5bda"),
"retries": "1",
"priority": "1",
"type": "email",
"data": "some.email#example.com"
}
]
}
]
});
let result = await conn.db.command({
"update": Group.collection.name,
"updates": [
{
"q": {},
"u": { "$set": { "people.$[i].contacts.$[j].data": "new data" } },
"multi": true,
"arrayFilters": [
{ "i._id": ObjectId("5a05a8c3e0ce3444f8ec5bd8") },
{ "j._id": ObjectId("5a05a8dee0ce3444f8ec5bda") }
]
}
]
});
log(result);
let group = await Group.findOne();
log(group);
} catch(e) {
console.error(e);
} finally {
mongoose.disconnect();
}
})()
Since that sends the "command" directly through to the server, we see the expected update does in fact take place:
Mongoose: groups.remove({}, {})
Mongoose: groups.insert({ name: 'support', _id: ObjectId("5a06557fb568aa0ad793c5e4"), people: [ { _id: ObjectId("5a05a8c3e0ce3444f8ec5bd8"), enabled: true, level: '1', name: 'Someone', contacts: [ { type: 'email', data: 'adifferent.email#example.com', _id: ObjectId("5a06557fb568aa0ad793c5e5") }, { _id: ObjectId("5a05a8dee0ce3444f8ec5bda"), retries: '1', priority: '1', type: 'email', data: 'some.email#example.com' } ] } ], __v: 0 })
{ n: 1,
nModified: 1,
opTime:
{ ts: Timestamp { _bsontype: 'Timestamp', low_: 3, high_: 1510364543 },
t: 24 },
electionId: 7fffffff0000000000000018,
ok: 1,
operationTime: Timestamp { _bsontype: 'Timestamp', low_: 3, high_: 1510364543 },
'$clusterTime':
{ clusterTime: Timestamp { _bsontype: 'Timestamp', low_: 3, high_: 1510364543 },
signature: { hash: [Object], keyId: 0 } } }
Mongoose: groups.findOne({}, { fields: {} })
{
"_id": "5a06557fb568aa0ad793c5e4",
"name": "support",
"__v": 0,
"people": [
{
"_id": "5a05a8c3e0ce3444f8ec5bd8",
"enabled": true,
"level": "1",
"name": "Someone",
"contacts": [
{
"type": "email",
"data": "adifferent.email#example.com",
"_id": "5a06557fb568aa0ad793c5e5"
},
{
"_id": "5a05a8dee0ce3444f8ec5bda",
"retries": "1",
"priority": "1",
"type": "email",
"data": "new data" // <-- updated here
}
]
}
]
}
So right "now"[1] the drivers available "off the shelf" don't actually implement .update() or it's other implementing counterparts in a way that is compatible with actually passing through the necessary arrayFilters argument. So if you are "playing with" a development series or release candiate server, then you really should be prepared to be working with the "bleeding edge" and unreleased drivers as well.
But you can actually do this as demonstrated in any driver, in the correct form where the command being issued is not going to be altered.
[1] As of writing on November 11th 2017 there is no "official" release of MongoDB or the supported drivers that actually implement this. Production usage should be based on official releases of the server and supported drivers only.
I had a similar use case. But my second level nested array doesn't have a key. While most examples out there showcase an example with arrays having a key like this:
{
"id": 1,
"items": [
{
"name": "Product 1",
"colors": ["yellow", "blue", "black"]
}
]
}
My use case is like this, without the key:
{
"colors": [
["yellow"],
["blue"],
["black"]
]
}
I managed to use the arrayfilters by ommiting the label of the first level of the array nest. Example document:
db.createCollection('ProductFlow')
db.ProductFlow.insertOne(
{
"steps": [
[
{
"actionType": "dispatch",
"payload": {
"vehicle": {
"name": "Livestock Truck",
"type": "road",
"thirdParty": true
}
}
},
{
"actionType": "dispatch",
"payload": {
"vehicle": {
"name": "Airplane",
"type": "air",
"thirdParty": true
}
}
}
],
[
{
"actionType": "store",
"payload": {
"company": "Company A",
"is_supplier": false
}
}
],
[
{
"actionType": "sell",
"payload": {
"reseller": "Company B",
"is_supplier": false
}
}
]
]
}
)
In my case, I want to:
Find all documents that have any steps with payload.vehicle.thirdParty=true and actionType=dispatch
Update the actions set payload.vehicle.thirdParty=true only for the actions that have actionType=dispatch.
My first approach was withour arrayfilters. But it would create the property payload.vehicle.thirdParty=true inside the steps with actionType store and sell.
The final query that updated the properties only inside the steps with actionType=dispatch:
Mongo Shell:
db.ProductFlow.updateMany(
{"steps": {"$elemMatch": {"$elemMatch": {"payload.vehicle.thirdParty": true, "actionType": "dispatch"}}}},
{"$set": {"steps.$[].$[i].payload.vehicle.thirdParty": false}},
{"arrayFilters": [ { "i.actionType": "dispatch" } ], multi: true}
)
PyMongo:
query = {
"steps": {"$elemMatch": {"$elemMatch": {"payload.vehicle.thirdParty": True, "actionType": "dispatch"}}}
}
update_statement = {
"$set": {
"steps.$[].$[i].payload.vehicle.thirdParty": False
}
}
array_filters = [
{ "i.actionType": "dispatch" }
]
NOTE that I'm omitting the label on the first array at the update statement steps.$[].$[i].payload.vehicle.thirdParty. Most examples out there will use both labels because their objects have a key for the array. I took me some time to figure that out.

Update's request for mongodb [duplicate]

I have a document structure that is deeply nested, like this:
{id: 1,
forecasts: [ {
forecast_id: 123,
name: "Forecast 1",
levels: [
{ level: "proven",
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
},
{ level: "likely",
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
}
]
},
]
}
I'm trying to update the collection to insert a new config, that looks like this:
newdata = {
config: "Custom 1",
variables: [{ x: 111, y:2222, z:3333}]
}
I'm trying something like this in mongo (in Python):
db.myCollection.update({"id": 1,
"forecasts.forecast-id": 123,
"forecasts.levels.level": "proven",
"forecasts.levels.configs.config": "Custom 1"
},
{"$set": {"forecasts.$.levels.$.configs.$": newData}}
)
I'm getting "Cannot apply the positional operator without a corresponding query field containing an array" error though. What is the proper way to do this in mongo? This is mongo v2.4.1.
Unfortunately, you can't use the $ operator more than once per key, so you have to use numeric values for the rest. As in:
db.myCollection.update({
"id": 1,
"forecasts.forecast-id": 123,
"forecasts.levels.level": "proven",
"forecasts.levels.configs.config": "Custom 1"
},
{"$set": {"forecasts.$.levels.0.configs.0": newData}}
)
MongoDB's support for updating nested arrays is poor. So you're best off avoiding their use if you need to update the data frequently, and consider using multiple collections instead.
One possibility: make forecasts its own collection, and assuming you have a fixed set of level values, make level an object instead of an array:
{
_id: 123,
parentId: 1,
name: "Forecast 1",
levels: {
proven: {
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
},
likely: {
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
}
}
}
Then you can update it using:
db.myCollection.update({
_id: 123,
'levels.proven.configs.config': 'Custom 1'
},
{ $set: { 'levels.proven.configs.$': newData }}
)
Managed to solve it with using mongoose:
All you need to know is the '_id's of all of the sub-document in the chain (mongoose automatically create '_id' for each sub-document).
for example -
SchemaName.findById(_id, function (e, data) {
if (e) console.log(e);
data.sub1.id(_id1).sub2.id(_id2).field = req.body.something;
// or if you want to change more then one field -
//=> var t = data.sub1.id(_id1).sub2.id(_id2);
//=> t.field = req.body.something;
data.save();
});
More about the sub-document _id method in mongoose documentation.
explanation:_id is for the SchemaName, _id1 for sub1 and _id2 for sub2 - you can keep chaining like that.
*You don't have to use findById method, but it's seem to me the most convenient as you need to know the rest of the '_id's anyway.
MongoDB has introduced ArrayFilters to tackle this issue in Version 3.5.2 and later.
New in version 3.6.
Starting in MongoDB 3.6, when updating an array field, you can specify
arrayFilters that determine which array elements to update.
[https://docs.mongodb.com/manual/reference/method/db.collection.update/#specify-arrayfilters-for-an-array-update-operations][1]
Let's say the Schema design as follows :
var ProfileSchema = new Schema({
name: String,
albums: [{
tour_name: String,
images: [{
title: String,
image: String
}]
}]
});
And Document created looks like this :
{
"_id": "1",
"albums": [{
"images": [
{
"title": "t1",
"url": "url1"
},
{
"title": "t2",
"url": "url2"
}
],
"tour_name": "london-trip"
},
{
"images": [.........]:
}]
}
Say I want to update the "url" of an image.
Given - "document id", "tour_name" and "title"
For this the update query :
Profiles.update({_id : req.body.id},
{
$set: {
'albums.$[i].images.$[j].title': req.body.new_name
}
},
{
arrayFilters: [
{
"i.tour_name": req.body.tour_name, "j.image": req.body.new_name // tour_name - current tour name, new_name - new tour name
}]
})
.then(function (resp) {
console.log(resp)
res.json({status: 'success', resp});
}).catch(function (err) {
console.log(err);
res.status(500).json('Failed');
})
This is a very OLD bug in MongoDB
https://jira.mongodb.org/browse/SERVER-831
I was facing same kind of problem today, and after lot of exploring on google/stackoverflow/github, I figured arrayFilters are the best solution to this problem. Which would work with mongo 3.6 and above.
This link finally saved my day: https://thecodebarbarian.com/a-nodejs-perspective-on-mongodb-36-array-filters.html
const OrganizationInformationSchema = mongoose.Schema({
user: {
_id: String,
name: String
},
organizations: [{
name: {
type: String,
unique: true,
sparse: true
},
rosters: [{
name: {
type: String
},
designation: {
type: String
}
}]
}]
}, {
timestamps: true
});
And using mongoose in express, updating the name of roster of given id.
const mongoose = require('mongoose');
const ControllerModel = require('../models/organizations.model.js');
module.exports = {
// Find one record from database and update.
findOneRosterAndUpdate: (req, res, next) => {
ControllerModel.updateOne({}, {
$set: {
"organizations.$[].rosters.$[i].name": req.body.name
}
}, {
arrayFilters: [
{ "i._id": mongoose.Types.ObjectId(req.params.id) }
]
}).then(response => {
res.send(response);
}).catch(err => {
res.status(500).send({
message: "Failed! record cannot be updated.",
err
});
});
}
}
It's fixed.
https://jira.mongodb.org/browse/SERVER-831
But this feature is available starting with the MongoDB 3.5.12 development version.
Note: This question asked on Aug 11 2013 and it's resolved on Aug 11 2017
Given how MongoDB doesn't appear to provide a good mechanism for this, I find it prudent to use mongoose to simply extract the element from the mongo collection using .findOne(...), run a for-loop search on its relevant subelements (seeking by say ObjectID), modify that JSON, then do Schema.markModified('your.subdocument'); Schema.save(); It's probably not efficient, but it is very simple and works fine.
I searched about this for about 5 hours and finally found the best and easiest solution:
HOW TO UPDATE NESTED SUB-DOCUMENTS IN MONGO DB
{id: 1,
forecasts: [ {
forecast_id: 123,
name: "Forecast 1",
levels: [
{
levelid:1221
levelname: "proven",
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
},
{
levelid:1221
levelname: "likely",
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
}
]
},
]}
Query:
db.weather.updateOne({
"_id": ObjectId("1"), //this is level O select
"forecasts": {
"$elemMatch": {
"forecast_id": ObjectId("123"), //this is level one select
"levels.levelid": ObjectId("1221") // this is level to select
}
}
},
{
"$set": {
"forecasts.$[outer].levels.$[inner].levelname": "New proven",
}
},
{
"arrayFilters": [
{ "outer.forecast_id": ObjectId("123") },
{ "inner.levelid": ObjectId("1221") }
]
}).then((result) => {
resolve(result);
}, (err) => {
reject(err);
});
Sharing my lessons learned. I faced the same requirement recently where i need to update a nested array item.
My structure is as follows
{
"main": {
"id": "ID_001",
"name": "Fred flinstone Inc"
},
"types": [
{
"typeId": "TYPE1",
"locations": [
{
"name": "Sydney",
"units": [
{
"unitId": "PHG_BTG1"
}
]
},
{
"name": "Brisbane",
"units": [
{
"unitId": "PHG_KTN1"
},
{
"unitId": "PHG_KTN2"
}
]
}
]
}
]
}
My requirement is to add some fields in a specific units[].
My solution is first to find the index of the nested array item (say foundUnitIdx)
The two techniques I used are
use the $set keyword
specify the dynamic field in $set using the [] syntax
query = {
"locations.units.unitId": "PHG_KTN2"
};
var updateItem = {
$set: {
["locations.$.units."+ foundUnitIdx]: unitItem
}
};
var result = collection.update(
query,
updateItem,
{
upsert: true
}
);
Hope this helps others. :)
EASY SOLUTION FOR Mongodb 3.2+
https://docs.mongodb.com/manual/reference/method/db.collection.replaceOne/
I had a similar situation and solved it like this. I was using mongoose, but it should still work in vanilla MongoDB. Hope it's useful to someone.
const MyModel = require('./model.js')
const query = {id: 1}
// First get the doc
MyModel.findOne(query, (error, doc) => {
// Do some mutations
doc.foo.bar.etc = 'some new value'
// Pass in the mutated doc and replace
MyModel.replaceOne(query, doc, (error, newDoc) => {
console.log('It worked!')
})
}
Depending on your use case, you might be able to skip the initial findOne()
Okkk.we can update our nested subdocument in mongodb.this is our schema.
var Post = new mongoose.Schema({
name:String,
post:[{
like:String,
comment:[{
date:String,
username:String,
detail:{
time:String,
day:String
}
}]
}]
})
solution for this schema
Test.update({"post._id":"58206a6aa7b5b99e32b7eb58"},
{$set:{"post.$.comment.0.detail.time":"aajtk"}},
function(err,data){
//data is updated
})

Mongodb update deeply nested subdocument

I have a document structure that is deeply nested, like this:
{id: 1,
forecasts: [ {
forecast_id: 123,
name: "Forecast 1",
levels: [
{ level: "proven",
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
},
{ level: "likely",
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
}
]
},
]
}
I'm trying to update the collection to insert a new config, that looks like this:
newdata = {
config: "Custom 1",
variables: [{ x: 111, y:2222, z:3333}]
}
I'm trying something like this in mongo (in Python):
db.myCollection.update({"id": 1,
"forecasts.forecast-id": 123,
"forecasts.levels.level": "proven",
"forecasts.levels.configs.config": "Custom 1"
},
{"$set": {"forecasts.$.levels.$.configs.$": newData}}
)
I'm getting "Cannot apply the positional operator without a corresponding query field containing an array" error though. What is the proper way to do this in mongo? This is mongo v2.4.1.
Unfortunately, you can't use the $ operator more than once per key, so you have to use numeric values for the rest. As in:
db.myCollection.update({
"id": 1,
"forecasts.forecast-id": 123,
"forecasts.levels.level": "proven",
"forecasts.levels.configs.config": "Custom 1"
},
{"$set": {"forecasts.$.levels.0.configs.0": newData}}
)
MongoDB's support for updating nested arrays is poor. So you're best off avoiding their use if you need to update the data frequently, and consider using multiple collections instead.
One possibility: make forecasts its own collection, and assuming you have a fixed set of level values, make level an object instead of an array:
{
_id: 123,
parentId: 1,
name: "Forecast 1",
levels: {
proven: {
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
},
likely: {
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
}
}
}
Then you can update it using:
db.myCollection.update({
_id: 123,
'levels.proven.configs.config': 'Custom 1'
},
{ $set: { 'levels.proven.configs.$': newData }}
)
Managed to solve it with using mongoose:
All you need to know is the '_id's of all of the sub-document in the chain (mongoose automatically create '_id' for each sub-document).
for example -
SchemaName.findById(_id, function (e, data) {
if (e) console.log(e);
data.sub1.id(_id1).sub2.id(_id2).field = req.body.something;
// or if you want to change more then one field -
//=> var t = data.sub1.id(_id1).sub2.id(_id2);
//=> t.field = req.body.something;
data.save();
});
More about the sub-document _id method in mongoose documentation.
explanation:_id is for the SchemaName, _id1 for sub1 and _id2 for sub2 - you can keep chaining like that.
*You don't have to use findById method, but it's seem to me the most convenient as you need to know the rest of the '_id's anyway.
MongoDB has introduced ArrayFilters to tackle this issue in Version 3.5.2 and later.
New in version 3.6.
Starting in MongoDB 3.6, when updating an array field, you can specify
arrayFilters that determine which array elements to update.
[https://docs.mongodb.com/manual/reference/method/db.collection.update/#specify-arrayfilters-for-an-array-update-operations][1]
Let's say the Schema design as follows :
var ProfileSchema = new Schema({
name: String,
albums: [{
tour_name: String,
images: [{
title: String,
image: String
}]
}]
});
And Document created looks like this :
{
"_id": "1",
"albums": [{
"images": [
{
"title": "t1",
"url": "url1"
},
{
"title": "t2",
"url": "url2"
}
],
"tour_name": "london-trip"
},
{
"images": [.........]:
}]
}
Say I want to update the "url" of an image.
Given - "document id", "tour_name" and "title"
For this the update query :
Profiles.update({_id : req.body.id},
{
$set: {
'albums.$[i].images.$[j].title': req.body.new_name
}
},
{
arrayFilters: [
{
"i.tour_name": req.body.tour_name, "j.image": req.body.new_name // tour_name - current tour name, new_name - new tour name
}]
})
.then(function (resp) {
console.log(resp)
res.json({status: 'success', resp});
}).catch(function (err) {
console.log(err);
res.status(500).json('Failed');
})
This is a very OLD bug in MongoDB
https://jira.mongodb.org/browse/SERVER-831
I was facing same kind of problem today, and after lot of exploring on google/stackoverflow/github, I figured arrayFilters are the best solution to this problem. Which would work with mongo 3.6 and above.
This link finally saved my day: https://thecodebarbarian.com/a-nodejs-perspective-on-mongodb-36-array-filters.html
const OrganizationInformationSchema = mongoose.Schema({
user: {
_id: String,
name: String
},
organizations: [{
name: {
type: String,
unique: true,
sparse: true
},
rosters: [{
name: {
type: String
},
designation: {
type: String
}
}]
}]
}, {
timestamps: true
});
And using mongoose in express, updating the name of roster of given id.
const mongoose = require('mongoose');
const ControllerModel = require('../models/organizations.model.js');
module.exports = {
// Find one record from database and update.
findOneRosterAndUpdate: (req, res, next) => {
ControllerModel.updateOne({}, {
$set: {
"organizations.$[].rosters.$[i].name": req.body.name
}
}, {
arrayFilters: [
{ "i._id": mongoose.Types.ObjectId(req.params.id) }
]
}).then(response => {
res.send(response);
}).catch(err => {
res.status(500).send({
message: "Failed! record cannot be updated.",
err
});
});
}
}
It's fixed.
https://jira.mongodb.org/browse/SERVER-831
But this feature is available starting with the MongoDB 3.5.12 development version.
Note: This question asked on Aug 11 2013 and it's resolved on Aug 11 2017
Given how MongoDB doesn't appear to provide a good mechanism for this, I find it prudent to use mongoose to simply extract the element from the mongo collection using .findOne(...), run a for-loop search on its relevant subelements (seeking by say ObjectID), modify that JSON, then do Schema.markModified('your.subdocument'); Schema.save(); It's probably not efficient, but it is very simple and works fine.
I searched about this for about 5 hours and finally found the best and easiest solution:
HOW TO UPDATE NESTED SUB-DOCUMENTS IN MONGO DB
{id: 1,
forecasts: [ {
forecast_id: 123,
name: "Forecast 1",
levels: [
{
levelid:1221
levelname: "proven",
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
},
{
levelid:1221
levelname: "likely",
configs: [
{
config: "Custom 1",
variables: [{ x: 1, y:2, z:3}]
},
{
config: "Custom 2",
variables: [{ x: 10, y:20, z:30}]
},
]
}
]
},
]}
Query:
db.weather.updateOne({
"_id": ObjectId("1"), //this is level O select
"forecasts": {
"$elemMatch": {
"forecast_id": ObjectId("123"), //this is level one select
"levels.levelid": ObjectId("1221") // this is level to select
}
}
},
{
"$set": {
"forecasts.$[outer].levels.$[inner].levelname": "New proven",
}
},
{
"arrayFilters": [
{ "outer.forecast_id": ObjectId("123") },
{ "inner.levelid": ObjectId("1221") }
]
}).then((result) => {
resolve(result);
}, (err) => {
reject(err);
});
Sharing my lessons learned. I faced the same requirement recently where i need to update a nested array item.
My structure is as follows
{
"main": {
"id": "ID_001",
"name": "Fred flinstone Inc"
},
"types": [
{
"typeId": "TYPE1",
"locations": [
{
"name": "Sydney",
"units": [
{
"unitId": "PHG_BTG1"
}
]
},
{
"name": "Brisbane",
"units": [
{
"unitId": "PHG_KTN1"
},
{
"unitId": "PHG_KTN2"
}
]
}
]
}
]
}
My requirement is to add some fields in a specific units[].
My solution is first to find the index of the nested array item (say foundUnitIdx)
The two techniques I used are
use the $set keyword
specify the dynamic field in $set using the [] syntax
query = {
"locations.units.unitId": "PHG_KTN2"
};
var updateItem = {
$set: {
["locations.$.units."+ foundUnitIdx]: unitItem
}
};
var result = collection.update(
query,
updateItem,
{
upsert: true
}
);
Hope this helps others. :)
EASY SOLUTION FOR Mongodb 3.2+
https://docs.mongodb.com/manual/reference/method/db.collection.replaceOne/
I had a similar situation and solved it like this. I was using mongoose, but it should still work in vanilla MongoDB. Hope it's useful to someone.
const MyModel = require('./model.js')
const query = {id: 1}
// First get the doc
MyModel.findOne(query, (error, doc) => {
// Do some mutations
doc.foo.bar.etc = 'some new value'
// Pass in the mutated doc and replace
MyModel.replaceOne(query, doc, (error, newDoc) => {
console.log('It worked!')
})
}
Depending on your use case, you might be able to skip the initial findOne()
Okkk.we can update our nested subdocument in mongodb.this is our schema.
var Post = new mongoose.Schema({
name:String,
post:[{
like:String,
comment:[{
date:String,
username:String,
detail:{
time:String,
day:String
}
}]
}]
})
solution for this schema
Test.update({"post._id":"58206a6aa7b5b99e32b7eb58"},
{$set:{"post.$.comment.0.detail.time":"aajtk"}},
function(err,data){
//data is updated
})