I have a collection with around 500,0000 entries in MongoDB.
Each entry has an attribute Date in the following format:
"Date" : "21/01/2005"
I'd like to know how I can convert it in such a way to Date format, so I can then index it (old-new) and query for entries by year.
I have tried:
db.collection.find().forEach(function(element){
element.OrderDate = ISODate(element.OrderDate);
db.collection.save(element);
})
But this just seems to change the Date attribute to today's date, along with time in the following format:
"Date" : ISODate("2016-02-11T11:41:45.680Z")
Thank you in advance.
Convert the field to the correct date object by spliting the string on the given delimiter. Use parseInt() to convert the strings into numbers, and the new Date() constructor builds a Date from those parts: the third part will be the year, the second part the month, and the first part the day. Since Date uses zero-based month numbers you have to subtract one from the month number.
The following demonstrates this approach:
var cursor = db.collection.find({"OrderDate": {"$exists": true, "$type": 2 }});
while (cursor.hasNext()) {
var doc = cursor.next();
var parts = doc.OrderDate.split("/");
var dt = new Date(
parseInt(parts[2], 10), // year
parseInt(parts[1], 10) - 1, // month
parseInt(parts[0], 10) // day
);
db.collection.update(
{"_id": doc._id},
{"$set": {"OrderDate": dt}}
)
};
For improved performance especially when dealing with large collections, take advantage of using the Bulk API for bulk updates as you will be sending the operations to the server in batches of say 500 which gives you a better performance as you are not sending every request to the server, just once in every 500 requests.
The following demonstrates this approach, the first example uses the Bulk API available in MongoDB versions >= 2.6 and < 3.2. It updates all
the documents in the collection by changing the OrderDate fields to date fields:
var bulk = db.collection.initializeUnorderedBulkOp(),
counter = 0;
db.collection.find({"OrderDate": {"$exists": true, "$type": 2 }}).forEach(function (doc) {
var parts = doc.OrderDate.split("/");
var dt = new Date(
parseInt(parts[2], 10), // year
parseInt(parts[1], 10) - 1, // month
parseInt(parts[0], 10) // day
);
bulk.find({ "_id": doc._id }).updateOne({
"$set": { "OrderDate": dt}
});
counter++;
if (counter % 500 == 0) {
bulk.execute(); // Execute per 500 operations and re-initialize every 500 update statements
bulk = db.collection.initializeUnorderedBulkOp();
}
})
// Clean up remaining operations in queue
if (counter % 500 != 0) { bulk.execute(); }
The next example applies to the new MongoDB version 3.2 which has since deprecated the Bulk API and provided a newer set of apis using bulkWrite():
var bulkOps = db.collection.find({"OrderDate": {"$exists": true, "$type": 2 }}).map(function (doc) {
var parts = doc.OrderDate.split("/");
var dt = new Date(
parseInt(parts[2], 10), // year
parseInt(parts[1], 10) - 1, // month
parseInt(parts[0], 10) // day
);
return {
"updateOne": {
"filter": { "_id": doc._id } ,
"update": { "$set": { "OrderDate": dt } }
}
};
})
db.collection.bulkWrite(bulkOps);
Related
This Meteor server code tries to count all the records which are 4 months and newer with property size:'4', color:'white' but account all entires from any one user as one count, so no mater how many documents have been entered by the same user, the are all counted as one. but I am getting nothing in return. any ideas? thx
let date = new Date();
date.setMonth(date.getMonth() - 4);
let doc = UsageCol.aggregate([{
$match: {
createdAt: {
$gte: date,
$lte: new Date()
},
action: 'failBroadcast',
plate: plate
}
}, {
$group: {
_id: {
userId: "$userId"
},
count: {
$sum: 1
}
}
}]);
for (var i = 0; i < doc.length; i++) {
var obj = doc[i];
console.log(JSON.stringify(obj));
}
Alright I just wanted to clear some things up from this morning.
The only reason I recommended moment js was thinking we are storing the date in date type and there is no easy way to dynamically create date in UTC using java script date function
So now that we know you used Date.now() to save the dates, you don't need any moment js.
The correct syntax is
let dateToMillis = Date.now(); //The current millis from epoch.
let dateFrom = new Date(dateToMillis); // Use the current millis from epoch.
let dateFromMillis = dateFrom.setMonth(dateFrom.getMonth() - 4); // The millis 4 months ago from epoch.
Pass dateToMillis and dateFromMillis to aggregation query.
I've got a collection consisting of millions of documents that resemble the following:
{
_id: ObjectId('...'),
value: "0.53"
combo: [
{
h: 0,
v: "0.42"
},
{
h: 1,
v: "1.32"
}
]
}
The problem is that the values are stored as strings and I need to convert them to float/double.
I'm trying this and it's working but this'll take days to complete, given the volume of data:
db.collection.find({}).forEach(function(obj) {
if (typeof(obj.value) === "string") {
obj.value = parseFloat(obj.value);
db.collection.save(obj);
}
obj.combo.forEach(function(hv){
if (typeof(hv.value) === "string") {
hv.value = parseFloat(hv.value);
db.collection.save(obj);
}
});
});
I came across bulk update reading the Mongo docs and I'm trying this:
var bulk = db.collection.initializeUnorderedBulkOp();
bulk.find({}).update(
{
$set: {
"value": parseFloat("value"),
}
});
bulk.execute();
This runs... but I get a NAN as a value, which is because it thinks I'm trying to convert "value" to a float. I've tried different variations like this.value and "$value" but to no avail. Plus this approach only attempts to correct the value in the other object, not the ones in the array.
I'd appreciate any help. Thanks in advance!
Figured it out the following way:
1) To convert at the document level, I came across this post and the reply by Markus paved the way to my solution:
var bulk = db.collection.initializeUnorderedBulkOp()
var myDocs = db.collection.find()
var ops = 0
myDocs.forEach(
function(myDoc) {
bulk.find({ _id: myDoc._id }).updateOne(
{
$set : {
"value": parseFloat(myDoc.value),
}
}
);
if ((++ops % 1000) === 0){
bulk.execute();
bulk = db.collection.initializeUnorderedBulkOp();
}
}
)
bulk.execute();
2) The second part involved updating the array object values and I discovered the syntax to do so in the accepted answer on this post. In my case, I knew that there were 24 values in I ran this separately from the first query and the result looked like:
var bulk = db.collection.initializeUnorderedBulkOp()
var myDocs = db.collection.find()
var ops = 0
myDocs.forEach(
function(myDoc) {
bulk.find({ _id: myDoc._id }).update(
{
$set : {
"combo.0.v": parseFloat(myDoc.combo[0].v),
"combo.1.v": parseFloat(myDoc.combo[1].v),
"combo.2.v": parseFloat(myDoc.combo[2].v),
"combo.3.v": parseFloat(myDoc.combo[3].v),
"combo.4.v": parseFloat(myDoc.combo[4].v),
"combo.5.v": parseFloat(myDoc.combo[5].v),
"combo.6.v": parseFloat(myDoc.combo[6].v),
"combo.7.v": parseFloat(myDoc.combo[7].v),
"combo.8.v": parseFloat(myDoc.combo[8].v),
"combo.9.v": parseFloat(myDoc.combo[9].v),
"combo.10.v": parseFloat(myDoc.combo[10].v),
"combo.11.v": parseFloat(myDoc.combo[11].v),
"combo.12.v": parseFloat(myDoc.combo[12].v),
"combo.13.v": parseFloat(myDoc.combo[13].v),
"combo.14.v": parseFloat(myDoc.combo[14].v),
"combo.15.v": parseFloat(myDoc.combo[15].v),
"combo.16.v": parseFloat(myDoc.combo[16].v),
"combo.17.v": parseFloat(myDoc.combo[17].v),
"combo.18.v": parseFloat(myDoc.combo[18].v),
"combo.19.v": parseFloat(myDoc.combo[19].v),
"combo.20.v": parseFloat(myDoc.combo[20].v),
"combo.21.v": parseFloat(myDoc.combo[21].v),
"combo.22.v": parseFloat(myDoc.combo[22].v),
"combo.23.v": parseFloat(myDoc.combo[23].v)
}
}
);
if ((++ops % 1000) === 0){
bulk.execute();
bulk = db.collection.initializeUnorderedBulkOp();
}
}
)
bulk.execute();
Just to give an idea regarding performance, the forEach was going through around 900 documents a minute, which for 15 million records would have taken days, literally! Not only that but this was only converting the types at the document level, not the array level. For that, I would have to loop through each document and loop through each array (15 million x 24 iterations)! With this approach (running both queries side by side), it completed both in under 6 hours.
I hope this helps someone else.
I have my birthDate in string format like this "2010-03-22". I want to convert it in the Date type in MongoDB.
What db.patient.update() function should I write ?
I want to calculate the age of each person.
I used the solution give on How do I convert a property in MongoDB from text to date type? but all the dates got converted to "1970-01-01T00:00:00.000Z".
db.collection.find().forEach(function(e){
e.fieldname = new Date(e.fieldname)
db.collection.save(e)
});
If you are using robomonogo use new ISODate instead of new Date
One approach you could take in converting the field to the correct date object would be by splitting the string on the given delimiter "-". Use parseInt() to convert the delimited strings into numbers, and the new Date() constructor builds a Date from those parts: the first part will be the year, the second part the month, and the last part the day. Since Date uses zero-based month numbers you have to subtract one from the month number.
The following demonstrates this approach:
var cursor = db.patient.find({"birthDate": {"$exists": true, "$type": 2 }});
while (cursor.hasNext()) {
var doc = cursor.next();
var parts = doc.birthDate.split("-");
var dt = new Date(
parseInt(parts[0], 10), // year
parseInt(parts[1], 10) - 1, // month
parseInt(parts[2], 10) // day
);
db.patient.update(
{"_id": doc._id},
{"$set": {"birthDate": dt}}
)
};
For improved performance especially when dealing with large collections, take advantage of using the Bulk API for bulk updates as you will be sending the operations to the server in batches of say 500 which gives you a better performance as you are not sending every request to the server, just once in every 500 requests.
The following demonstrates this approach, the first example uses the Bulk API available in MongoDB versions >= 2.6 and < 3.2. It updates all
the documents in the collection by changing the OrderDate fields to date fields:
var bulk = db.patient.initializeUnorderedBulkOp(),
counter = 0;
db.patient.find({"birthDate": {"$exists": true, "$type": 2 }}).forEach(function (doc) {
var parts = doc.birthDate.split("-");
var dt = new Date(
parseInt(parts[0], 10), // year
parseInt(parts[1], 10) - 1, // month
parseInt(parts[2], 10) // day
);
bulk.find({ "_id": doc._id }).updateOne({
"$set": { "birthDate": dt}
});
counter++;
if (counter % 500 == 0) {
bulk.execute(); // Execute per 500 operations and re-initialize every 500 update statements
bulk = db.patient.initializeUnorderedBulkOp();
}
})
// Clean up remaining operations in queue
if (counter % 500 != 0) { bulk.execute(); }
The next example applies to the new MongoDB version 3.2 which has since deprecated the Bulk API and provided a newer set of apis using bulkWrite():
var bulkOps = db.patient.find({"birthDate": {"$exists": true, "$type": 2 }}).map(function (doc) {
var parts = doc.birthDate.split("-");
var dt = new Date(
parseInt(parts[0], 10), // year
parseInt(parts[1], 10) - 1, // month
parseInt(parts[2], 10) // day
);
return {
"updateOne": {
"filter": { "_id": doc._id } ,
"update": { "$set": { "birthDate": dt } }
}
};
})
db.patient.bulkWrite(bulkOps);
I have a database of events and I would like to find all events in a given month. The approach I'm taking is to find all events whose date is greater than, say February 1, 2015, and less than or equal to February 28, 2015. Here is the code I have:
var start = '2015-02-01T00:00:00:000Z';
var end = '2015-02-28T00:00:00:000Z';
Event
.find({
'start_date': {
'$gte' : start,
'$lte' : end
}
})
.sort({start_date: 1})
.exec(function (err, data) {
if (err) {
console.log('ERROR = ' + err);
} else {
month = data;
console.log('month' = JSON.stringify(month));
... // do other stuff
}
});
My query always comes back empty, even though I have records in my database that should be returned. What am I doing wrong?
UPDATE--------------------------------------------------------
Here is the schema I'm using:
var eventSchema = mongoose.Schema({
title : String,
details : String,
start_date : Date,
end_date : Date,
created : {
type: Date,
default: Date.now
}
});
A good explanation of how to get around this issue can be found in the article Querying for a Date Range (Specific Month or Day) which essentially suggests that you construct Date instances representing the start and end of the month used to do the range query (bearing in mind that in the Date() constructor, the month argument is 0-based index i.e. starts counting at 0, not 1. On the other hand, the days start counting at 1):
var start = new Date(2015, 1, 1);
var end = new Date(2015, 2, 1);
Event.find({
'start_date': {
'$gte' : start,
'$lt' : end
}
})
.sort({start_date: 1})
.exec(function (err, data) {
if (err) {
console.log('ERROR = ' + err);
} else {
month = data;
console.log('month' = JSON.stringify(month));
... // do other stuff
}
});
You could also try querying using the ISODate and see if it makes a difference:
'start_date': {
$gte: ISODate("2015-02-01T00:00:00.000Z"),
$lt: ISODate("2015-02-28T00:00:00.000Z")
}
Trying to figure out how to do this. Basically I want to sort by Hour / Day / Month / Year of my submissions.
Each submission has a created field which contains a Mongoose Date object in the form of "created" : ISODate("2013-03-11T01:49:09.421Z"). Do I need to compare against this in the find() conditions?
Here is my current query (I'm wrapping it in a count for pagination purposes FWIW so just ignore that part):
getSubmissionCount({}, function(count) {
// Sort by the range
switch (range) {
case 'today':
range = now.getTime();
case 'week':
range = now.getTime() - 7;
case 'month':
range = now.getTime() - 31; // TODO: make this find the current month and # of days in it
case 'year':
range = now.getTime() - 365;
case 'default':
range = now.getTime();
}
Submission.find({
}).skip(skip)
.sort('score', 'descending')
.sort('created', 'descending')
.limit(limit)
.execFind(function(err, submissions) {
if (err) {
callback(err);
}
if (submissions) {
callback(null, submissions, count);
}
});
});
Can someone help me figure this out? With that current code it just gives me all submissions regardless of a time range, so I'm obviously not doing something properly
I think, you are looking $lt(Less than) and $gt(Greater Than) operators in MongoDB.
By using above operators the result can be queried according to time.
I am adding possible solution below.
var d = new Date(),
hour = d.getHours(),
min = d.getMinutes(),
month = d.getMonth(),
year = d.getFullYear(),
sec = d.getSeconds(),
day = d.getDate();
Submission.find({
/* First Case: Hour */
created: { $lt: new Date(), $gt: new Date(year+','+month+','+day+','+hour+','+min+','+sec) } // Get results from start of current hour to current time.
/* Second Case: Day */
created: { $lt: new Date(), $gt: new Date(year+','+month+','+day) } // Get results from start of current day to current time.
/* Third Case: Month */
created: { $lt: new Date(), $gt: new Date(year+','+month) } // Get results from start of current month to current time.
/* Fourth Case: Year */
created: { $lt: new Date(), $gt: new Date(year) } // Get results from start of current year to current time.
})