Mongo: Upsert to copy all static fields to new document atomically - mongodb

I'm using the Bucket Pattern to limit documents' array size to maxBucketSize elements. Once a document's array elements is full (bucketSize = maxBucketSize), the next update will create a new document with a new array to hold more elements using upsert.
How would you copy the static fields (see below recordType and recordDesc) from the last full bucket with a single call?
Sample document:
// records collection
{
recordId: 12345, // non-unique index
recordType: "someType",
recordDesc: "Some record description.",
elements: [ { a: 1, b: 2 }, { a: 3, b: 4 } ]
bucketsize: 2,
}
Bucket implementation (copies only queried fields):
const maxBucketSize = 2;
db.collection('records').updateOne(
{
recordId: 12345,
bucketSize: { $lt: maxBucketSize } // false
},
{
$push: { elements: { a: 5, b: 6 } },
$inc: { bucketSize: 1 },
$setOnInsert: {
// Should be executed because bucketSize condition is false
// but fields `recordType` and `recordDesc` inaccessible as
// no documents matched and were not included in the query
}
},
{ upsert: true }
)
Possible solution
To make this work, I can always make two calls, findOne() to get static values and then updateOne() where I set fields with setOnInsert, but it's inefficient
How can I modify this as one call with an aggregate? Examining one (last added) document matching recordId (index), evaluate if array is full, and add new document.
Attempt:
// Evaluate last document added
db.collection('records').findOneAndUpdate(
{ recordId: 12345 },
{
$push: { elements: {
$cond: {
if: { $lt: [ '$bucketSize', maxBucketSize ] },
then: { a: 5, b: 6 }, else: null
}
}},
$inc: { bucketSize: {
$cond: {
if: { $lt: [ '$bucketSize', maxBucketSize ] },
then: 1, else: 0
}
}},
$setOnInsert: {
recordType: '$recordType',
recordDesc: '$recordDesc'
}
},
{
sort: { $natural: -1 }, // last document
upsert: true, // Bucket overflow
}
)
This comes back with:
MongoError: Cannot increment with non-numeric argument: { bucketSize: { $cond: { if: { $lt: [ "$bucketSize", 2 ] }, then: 1, else: 0 } }}

Related

MongoDB - Update the value of one field with the value of another nested field

I am trying to run a MongoDB query to update the value of one field with the value of another nested field. I have the following document:
{
"name": "name",
"address": "address",
"times": 10,
"snapshots": [
{
"dayTotal": 2,
"dayHit": 2,
"dayIndex": 2
},
{
"dayTotal": 3,
"dayHit": 3,
"dayIndex": 3
}
]
}
I am trying like this:
db.netGraphMetadataDTO.updateMany(
{ },
[{ $set: { times: "$snapshots.$[elem].dayTotal" } }],
{
arrayFilters: [{"elem.dayIndex":{"$eq": 2}}],
upsert: false,
multi: true
}
);
but got an error:
arrayFilters may not be specified for pipeline-syle updates
You can't use arrayFilters with aggregation pipeline for update query at the same time.
Instead, what you need to do:
Get the dayTotal field from the result 2.
Take the first matched document from the result 3.
Filter the document from snapshots array.
db.netGraphMetadataDTO.updateMany({},
[
{
$set: {
times: {
$getField: {
field: "dayTotal",
input: {
$first: {
$filter: {
input: "$snapshots",
cond: {
$eq: [
"$$this.dayIndex",
2
]
}
}
}
}
}
}
}
}
],
{
upsert: false,
multi: true
})
Demo # Mongo Playground

MongoDB: How to speed up my data reorganisation query/operation?

I'm trying to analyse some data and I thought my queries would be faster ultimately by storing a relationship between my collections instead. So I wrote something to do the data normalisation, which is as follows:
var count = 0;
db.Interest.find({'PersonID':{$exists: false}, 'Data.DateOfBirth': {$ne: null}})
.toArray()
.forEach(function (x) {
if (null != x.Data.DateOfBirth) {
var peep = { 'Name': x.Data.Name, 'BirthMonth' :x.Data.DateOfBirth.Month, 'BirthYear' :x.Data.DateOfBirth.Year};
var person = db.People.findOne(peep);
if (null == person) {
peep._id = db.People.insertOne(peep).insertedId;
//print(peep._id);
}
db.Interest.updateOne({ '_id': x._id }, {$set: { 'PersonID':peep._id }})
++count;
if ((count % 1000) == 0) {
print(count + ' updated');
}
}
})
This script is just passed to mongo.exe.
Basically, I attempt to find an existing person, if they don't exist create them. In either case, link the originating record with the individual person.
However this is very slow! There's about 10 million documents and at the current rate it will take about 5 days to complete.
Can I speed this up simply? I know I can multithread it to cut it down, but have I missed something?
In order to insert new persons into People collection, use this one:
db.Interest.aggregate([
{
$project: {
Name: "$Data.Name",
BirthMonth: "$Data.DateOfBirth.Month",
BirthYear: "$Data.DateOfBirth.Year",
_id: 0
}
},
{
$merge: {
into: "People",
// requires an unique index on {Name: 1, BirthMonth: 1, BirthYear: 1}
on: ["Name", "BirthMonth", "BirthYear"]
}
}
])
For updating PersonID in Interest collection use this pipeline:
db.Interest.aggregate([
{
$lookup: {
from: "People",
let: {
name: "$Data.Name",
month: "$Data.DateOfBirth.Month",
year: "$Data.DateOfBirth.Year"
},
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ["$Name", "$$name"] },
{ $eq: ["$BirthMonth", "$$month"] },
{ $eq: ["$BirthYear", "$$year"] }
]
}
}
},
{ $project: { _id: 1 } }
],
as: "interests"
}
},
{
$set: {
PersonID: { $first: "$interests._id" },
interests: "$$REMOVE"
}
},
{ $merge: { into: "Interest" } }
])
Mongo Playground

updateOne - filter one field, push or update sub field, add/update counter

I want to track tags frequency within a period. How do I update (counter, updateDate) the array if found, otherwise push to field?
const date = new Date();
const beginMonth = new Date(date.getFullYear(), date.getMonth(), 1);
await Tags.updateOne(
{ tag },
{
$set: {
tag,
period: {
// should have $cond: [{ $in: [beginMonth, "$period.createDate"] }]
$push: {
"createDate": beginMonth,
"updateDate": date,
"counter": 1,
}
}
}
},
{ upsert: true }
)
it should have a condition to check if { $in: [beginMonth, "$period.createDate"] } then {$inc: {counter:1} else $push. Also read that upsert cannot be used like this. How do I achieve this?
example:
If world tag exists and createDate is in beginMonth (August 1, 2021), then update counter to 2 and updateDate.
Let food tag does not exists, so it would push createDate,updateDate,counter
Before update:
{
tag: 'world',
period: [{
createDate: 2021-08-01T00:00:00.000+00:00,
updateDate:2021-08-28T13:16:58.508+00:00,
counter:1
}]
}
After update:
[{
tag: 'world',
period: [{
createDate: 2021-08-01T00:00:00.000+00:00,
updateDate:2021-08-28T14:16:58.508+00:00,
counter:2
}]
}
{
tag: 'food',
period: [{
createDate: 2021-08-01T00:00:00.000+00:00,
updateDate:2021-08-28T14:16:58.508+00:00,
counter:1
}]
}]
createDate probably bad naming, should be beginningDate. I am trying to tally tag frequency by month and keeping track of latest update for this period.
You can try update with aggregation pipeline starting from MongoDB 4.2,
$ifNull to check input field is null then return a blank array
$cond to check beginMonth is in array period.createDate
condition is true then,
$add to increment number in counter
$map to iterate loop of period array and check condition if beginMonth is period.createDate then increment counter otherwise nothing
$mergeObjects to merge current object with updated counter property
else,
$concatArrays to concat current period array with new input object
upsert will work because we have handled all the possible scenarios
const date = new Date();
const beginMonth = new Date(date.getFullYear(), date.getMonth(), 1);
await Tags.updateOne(
{ tag },
[{
$set: {
period: {
$cond: [
{ $in: [beginMonth, { $ifNull: ["$period.createDate", []] }] },
{
$map: {
input: "$period",
in: {
$mergeObjects: [
"$$this",
{
$cond: [
{ $eq: [beginMonth, "$$this.createDate"] },
{ counter: { $add: ["$$this.counter", 1] } },
{}
]
}
]
}
}
},
{
$concatArrays: [
{ $ifNull: ["$period", []] },
[
{
createDate: beginMonth,
updateDate: date,
counter: 1
}
]
]
}
]
}
}
}],
{ upsert: true }
)
Case 1: tag exists, date exists, so incremenet counter
Playground
Case 2: tag exists, date not exists, so add the new element in period array
Playground
Case 3: tag not exists, so add new document, this is the job of upsert
Playground

Update or append to a subcollection in mongo

I have a collection containing a subcollection. In one request, I would like to update a record in the subcollection or append to it if a match doesn't exist. For a bonus point I would also like this update to be a merge rather than an overwrite.
A crude example:
// Schema
{
subColl: [
{
name: String,
value: Number,
other: Number,
},
];
}
// Existing record
{
_id : 123,
subColl: [
{name: 'John',
value: 10,
other: 20}
]
}
// example
const update = { _id: 123, name: 'John', other: 1000 };
const { _id, name, other } = update;
const doc = await Schema.findById(_id);
const idx = doc.subColl.findIndex(({ name: nameInDoc }) => nameInDoc === name);
if (idx >= 0) {
doc.subColl[idx] = { ...doc.subColl[idx], other };
} else {
doc.subColl.push({ name, other });
}
doc.save();
Currently I can achieve this result by pulling the record, and doing the update/append manually but I am assuming that achieving it with a pure mongo query would be much faster.
I have tried:
Schema.findOneAndUpdate(
{ _id: 123, 'subColl.name': 'John' },
{ $set: { 'subColl.$': [{ name: 'John', other: 1000 }] } }
)
but this won't handle the append behaviour and also doesn't merge the object with the existing record, rather it overwrites it completely.
I am not sure is there any straight way to do this in single query,
Update with aggregation pipeline starting from MongoDB v4.2,
$cond to check name is in subColl array,
true condition, need to merge with existing object, $map to iterate loop, check condition if matches condition then merge new data object with current object using $mergeObjects
false condition, need to concat arrays, current subColl array and new object using $concatArrays
const _id = 123;
const update = { name: 'John', other: 1000 };
Schema.findOneAndUpdate(
{ _id: _id },
[{
$set: {
subColl: {
$cond: [
{ $in: [update.name, "$subColl.name"] },
{
$map: {
input: "$subColl",
in: {
$cond: [
{ $eq: ["$$this.name", update.name] },
{ $mergeObjects: ["$$this", update] },
"$$this"
]
}
}
},
{ $concatArrays: ["$subColl", [update]] }
]
}
}
}]
)
Playground

How to update with $arrayFilters for array variable value in Mongoose

By using arrayFilters, I need to update a collection with dynamic values from an array.
This is to say that, I need to update with array values in $inc, based on the array which is supplied in the arrayFilters.
SAMPLE DOCUMENT
{
_id:'shopId',
products:[
_id:'productId',
buys:[
{
_id:'abc',
quantity:0
},
{
_id:'def',
quantity:2
}
]
]
}
EXPECTED RESULT
{
_id:'shopId',
products:[
_id:'productId',
buys:[
{
_id:'abc',
quantity:1 //got incremented by 1
},
{
_id:'def',
quantity:5 //got incremented by 3
}
]
]
}
Consider this query here below for what I have tried
Shop.findOneAndUpdate(
{ "_id": ObjectId(shopId)},
{ $inc: { "products.$[p].buys.$[n].quantity": { $each: [1,3] } }},
{ arrayFilters: [{ "p._id": ObjectId('productId') }, { "n._id": {$in: ["abc", "def"]} }]
)
I need the n._id:'abc' to be incremented by 1 and n._id:'def' to be incremented by 3 which are put in an array (in that $each in $inc).
Now this doesn't work and gives out an error. I have tried replacing $each:[1,3] with $in: [1,3] but still doesn't work.
How can I do this?
As of MongoDB v4.2 it's not possible to do this with arrayFilters in a single operation.
I suggest updating them separately.
// Solution implemented in node.js since, you have tagged mongoose
// prepare condition data structure to use in multiple updates
const updateCondition = { "abc": 1, "def": 3 }
const updates = Object.entries(updateCondition).map(([key, value]) =>
Shop.findOneAndUpdate(
{ _id: ObjectId(shopId)} ,
{ $inc: { "products.$[p].buys.$[n].quantity": value } }, // value will be iterated through [1, 3]
{ arrayFilters: [{ "p._id": ObjectId(productId) }, { "n._id": key } }] // key will be iterated through ["abc", "def"]
).exec()
)
await Promise.all(updates).then(([resultA, resultB]) => {
// resultA is the result of "abc" update
// resultB is the result of "def" update
}).catch(err => {
// handle err
})