I'm currently having issues when querying for one of my Documents inside a Database through Meteor.
Using this line of code I'm trying to retrieve the next sequence number out of the DB. But it sometimes skips numbers randomly for some reason.
var col = MyCounters.findOne(type);
MyCounters.update(col._id, {$inc: {seq: 1}});
return col.seq;
Not getting any kind of errors server side.
Does anybody know what the issue might be?
I'm on Meteor 1.4+
====================
Update
I also update another Collection with the new value obtained from MyCounters collection, so it would be something like this:
var col = MyCounters.findOne(type);
MyCounters.update(col._id, {$inc: {seq: 1}});
var barId = col.seq;
// declare barObject + onInsertError
barObject.barId = barId;
// ...
FooCollection.insert(barObject, onInsertError);
And FooCollection ends up having skipped sequence numbers up to 5000 sometimes.
If you want increament at that item Document, you can use :
var col = MyCounters.findOne(type);
var valueOne = 1;
var nameItem = 'seq';
var inc = {};
inc[ nameItem ] = valueOne;
MyCounters.update({ _id: col._id }, { '$inc': inc } )
But if you want increament value from all Document from Collections MyCounters ( maks seq + 1 ) you can use :
var count = MyCounters.findOne({}, {sort:{seq:-1}}).seq;
count = count + 1;
MyCounters.update({_id:col._id}, {$set:{seq:count}})
I hope it work for you. Thanks
refer to : https://stackoverflow.com/a/33968766/4287229
Related
I try to abort a query after it works specific time. So ı use mongodb maxTimeMS() method for my query. But it is not working.
I except that this query will stop after 5 second but keep working. how ı terminate my query after it works specific time
var sum = 0 ;
var inbox=db.getMongo().getDB("xxxx").getCollection("Inbox");
var oldInbox=db.getMongo().getDB("xxxx").getCollection("oldInbox");
var inboxCount = inbox.count();
var oldCount = oldInbox.count();
var dif = inboxCount - oldCount ;
if ( dif > 10000000 ){
var cursor = inbox.find().maxTimeMS(5000);
cursor.sort({_id : -1}).forEach(function(uidDoc) {
var uid = uidDoc.uid;
var docsToMigrate = [];
var idsToRemove = [];
inbox.find({uid : uid}).sort({_id : -1}).skip(10).forEach(function(doc) {
docsToMigrate.push(doc);
idsToRemove.push(doc._id);
var x = doc._id;
});
oldInbox.insert(docsToMigrate);
inbox.remove({_id : {$in : idsToRemove}});
sum = sum + idsToRemove.length;
if ( x = 0 )
{
print(sum);
cursor.close();
}
});
};
Based on documentation here, maxTimeMS is a limit on processing time within the database. This will cover query planning, execution, and data serialization, but the timer stops when the cursor is idle. This is not wall time and means that it does not account for the execution time within your forEach callback. Since your query seems quite simple, it will likely take much more time than the maxTimeMS limit for it to stop.
So to answer your question, you'll need to keep a timer elsewhere and manage it yourself using paginated queries.
I have a Collection with a nested attribute that is an array of ObjectId References. These refer to documents in another Collection.
I'd like to replace these references with the documents themselves, i.e. embed those documents where the references are now. I've tried with and without the .snapshot() option. This may be caused because I'm updating a document while in a loop on that doc, and .snapshot() isn't available at that level.
My mongo-fu is low and I'm stuck on a call stack error. How can I do this?
Example code:
db.CollWithReferences.find({}).snapshot().forEach( function(document) {
var doc_id = document._id;
document.GroupsOfStuff.forEach( function(Group) {
var docsToEmbed= db.CollOfThingsToEmbed.find({ _id: { $in: Group.ArrayOfReferenceObjectIds }});
db.CollWithReferences.update({"_id": ObjectId(doc_id) },
{$set: {"Group.ArrayOfReferenceObjectIds ":docsToEmbed}} )
});
});
Gives this error:
{
"message" : "Maximum call stack size exceeded",
"stack" : "RangeError: Maximum call stack size exceeded" +
....}
I figure this is happening for one of two reasons. Either you are running out of memory by executing two queries in a for loop, or the update operation is being executed before the find operation has finished.
Either way, it is not a good idea to execute too many queries in a for loop as it can lead to this type of error.
I can't be sure if this will fix your problem as I don't know how many documents are in your collections, but it may work if you first get all documents from the CollWithReferences collection, then all you need from the CollOfThingsToEmbed collection. Then build a map of an _id from the CollOfThingsToEmbed collection to the actual document that corresponds to that. You can then loop through each document you got from the CollWithReferences collection, and mutate the groupsOfStuff array by accessing each ArrayOfReferenceObjectIds array and setting the ObjectId to the value that you have in the map you already built up, which will be the whole document. Then just update that document by setting GroupsOfSuff to its mutated value.
The following JavaScript code will do this (it could be organised better to have no logic in the global scope etc.):
var references = db.CollWithReferences.find({});
function getReferenceIds(references) {
var referenceIds = [];
for (var i = 0; i < references.length; i++) {
var group = references[i].GroupsOfStuff;
for (let j = 0; j < group.ArrayOfReferenceObjectIds; j++) {
referenceIds.push(group.ArrayOfReferenceObjectIds[j]);
}
}
return referenceIds;
}
function buildIdMap(docs) {
var map = {};
for (var i = 0; i < docs.length; i++) {
map[docs[i]._id.toString()] = docs[i];
}
return map;
}
var referenceIds = getReferenceIds(references);
var docsToEmbed = db.CollOfThingsToEmbed.find({_id: {$in: referenceIds}});
var idMap = buildIdMap(docsToEmbed);
for (var i = 0; i < references.length; i++) {
var groups = references[i].GroupsOfStuff;
for (var j = 0; j < groups.length; j++) {
refs = groups[j].ArrayOfReferenceObjectIds;
refs.forEach(function(ref) {
ref = idMap[ref.toString()];
});
}
db.CollWithReferences.update({
_id: ObjectId(ref._id)
}, {
$set: {GroupsOfStuff: groups}
});
}
It would be better if it was possible to just do one bulk update, but as each document needs to be updated differently, this is not possible.
My test collection has 56 entities in them. When the following script is executed the resulting collection has less entries than the original collection. The number varies for each run. What would cause this issue and is there a workaround for this?
var collectionToUpdate = 'testcollection';
var temporaryCollectionName = collectionToUpdate + '_old'
db.getCollection(collectionToUpdate).renameCollection(temporaryCollectionName);
var oldCollection = db.getCollection(temporaryCollectionName);
db.createCollection(collectionToUpdate);
var newCollection = db.getCollection(collectionToUpdate);
var count = 0;
oldCollection.find().forEach(
function (element) {
count++;
newCollection.insert(element)
}
);
print(count);
Versions used:
MongoDB - 3.2.8
RoboMongo - 0.9.0-RC10
I am trying to write a simpler session (haskell driver if it matters) with mongodb backend. I may be wrong but it seems a bit slow compared to when I run the bench without the session.
With session it gives me 25 connections a second - 10596 without.
Once the session is set on the initial load, all it does is compares the SID from cookie to the SID stored in session document in mongodb. So on every request it does a single trip to the database server. I get the SID from cookie and check if a document with such SID exist in mongodb. That's all. I am learning, so my session logic could be off too.
At the moment, I use count to check if the document exist. I count documents with relevant SID and test if it == 1. Is this a fast enough way to check if document exist?
I found in this document test if document exists that testing with find and limit is faster. But it only compares it to findOne - not to count.
So my question is: what is the fastest way to check if a document exist?
Thanks.
As to your question, have a look at the source code of find/findOne/count
rs0:PRIMARY> db.geo.count
function ( x ){
return this.find( x ).count();
}
rs0:PRIMARY> db.geo.findOne
function ( query , fields, options ){
var cursor = this.find(query, fields, -1 /* limit */, 0 /* skip*/,
0 /* batchSize */, options);
if ( ! cursor.hasNext() )
return null;
var ret = cursor.next();
if ( cursor.hasNext() ) throw "findOne has more than 1 result!";
if ( ret.$err )
throw "error " + tojson( ret );
return ret;
}
rs0:PRIMARY> db.geo.find
function ( query , fields , limit , skip, batchSize, options ){
var cursor = new DBQuery( this._mongo , this._db , this ,
this._fullName , this._massageObject( query ) , fields , limit , skip , batchSize , options || this.getQueryOptions() );
var connObj = this.getMongo();
var readPrefMode = connObj.getReadPrefMode();
if (readPrefMode != null) {
cursor.readPref(readPrefMode, connObj.getReadPrefTagSet());
}
return cursor;
}
The difference is, findOne/count uses something from this.find, while find uses DBQuery.
So I did a benchmark on the 3 ways:
function benchMark1() {
var date = new Date();
for (var i = 0; i < 100000; i++) {
db.zips.find({
"_id": "35004"
}, {
_id: 1
});
}
print(new Date() - date);
}
function benchMark2() {
var date = new Date();
for (var i = 0; i < 100000; i++) {
db.zips.findOne({
"_id": "35004"
}, {
_id: 1
});
}
print(new Date() - date);
}
function benchMark3() {
var date = new Date();
for (var i = 0; i < 100000; i++) {
db.zips.count({
"_id": "35004"
}, {
_id: 1
});
}
print(new Date() - date);
}
It turns out benchMark1 takes 1046ms, 2 takes 37611ms, 3 takes 63306ms.
It seems you are using the worst one.
EDIT: The reason why it's slow is described here: https://dba.stackexchange.com/questions/7573/difference-between-mongodbs-find-and-findone-calls
What else, make sure you have an unique index on the field SID:
rs0:PRIMARY> db.system.indexes.find()
If no index exists on SID,
rs0:PRIMARY> db.session.ensureIndex({SID: 1}, {unique: true}) // change "session" to your collection name
Note that although _id is usually an ObjectId, it doesn't have to be. So you can use the SID as _id. And there's already an index on it so that you can save an index and thus make the insertion faster. To do this, just set the _id field to SID when you insert a record.
{
_id: [value of SID]
... // rest of record
}
And if this still doesn't meat your requirements, you need to try analyse where the bottleneck is. That's another topic we can talk about if necessary.
I wrote a mapreduce function where the records are emitted in the following format
{userid:<xyz>, {event:adduser, count:1}}
{userid:<xyz>, {event:login, count:1}}
{userid:<xyz>, {event:login, count:1}}
{userid:<abc>, {event:adduser, count:1}}
where userid is the key and the remaining are the value for that key.
After the MapReduce function, I want to get the result in following format
{userid:<xyz>,{events: [{adduser:1},{login:2}], allEventCount:3}}
To acheive this I wrote the following reduce function
I know this can be achieved by group by.. both in aggregation framework and mapreduce, but we require a similar functionality for a complex scenario. So, I am taking this approach.
var reducefn = function(key,values){
var result = {allEventCount:0, events:[]};
values.forEach(function(value){
var notfound=true;
for(var n = 0; n < result.events.length; n++){
eventObj = result.events[n];
for(ev in eventObj){
if(ev==value.event){
result.events[n][ev] += value.allEventCount;
notfound=false;
break;
}
}
}
if(notfound==true){
var newEvent={}
newEvent[value.event]=1;
result.events.push(newEvent);
}
result.allEventCount += value.allEventCount;
});
return result;
}
This runs perfectly, when I run for 1000 records, when there are 3k or 10k records, the result I get is something like this
{ "_id" : {...}, "value" :{"allEventCount" :30, "events" :[ { "undefined" : 1},
{"adduser" : 1 }, {"remove" : 3 }, {"training" : 1 }, {"adminlogin" : 1 },
{"downgrade" : 2 } ]} }
Not able to understand where this undefined came from and also the sum of the individual events is less than allEventCount. All the docs in the collection has non-empty field event so there is no chance of undefined.
Mongo DB version -- 2.2.1
Environment -- Local machine, no sharding.
In the reduce function, why should this operation fail result.events[n][ev] += value.allEventCount; when the similar operation result.allEventCount += value.allEventCount; passes?
The corrected answer as suggested by johnyHK
Reduce function:
var reducefn = function(key,values){
var result = {totEvents:0, event:[]};
values.forEach(function(value){
value.event.forEach(function(eventElem){
var notfound=true;
for(var n = 0; n < result.event.length; n++){
eventObj = result.event[n];
for(ev in eventObj){
for(evv in eventElem){
if(ev==evv){
result.event[n][ev] += eventElem[evv];
notfound=false;
break;
}
}}
}
if(notfound==true){
result.event.push(eventElem);
}
});
result.totEvents += value.totEvents;
});
return result;
}
The shape of the object you emit from your map function must be the same as the object returned from your reduce function, as the results of a reduce can get fed back into reduce when processing large numbers of docs (like in this case).
So you need to change your emit to emit docs like this:
{userid:<xyz>, {events:[{adduser: 1}], allEventCount:1}}
{userid:<xyz>, {events:[{login: 1}], allEventCount:1}}
and then update your reduce function accordingly.