Find document range same day from mongodb use GetMongo processor by NiFi - date

I want to get the documents in mongodb for the day. I tried querying with mongo compass the code works. The code is as follows:
{
"time_created": {
"$gte": new Date(new Date().setHours(0, 0, 0, 0)),
"$lt": new Date(new Date().setHours(23, 59, 59, 999))
}
}
But when I use the above query code with the "query" property of GetMongo, I get the following error message:
Verification Results
Perform Validation
Component is invalid: 'Query' validated against '{ "time_created": { "$gte": new Date(new Date().setHours(0, 0, 0, 0)), "$lt": new Date(new Date().setHours(23, 59, 59, 999)) } }' is invalid because Query is not a valid JSON representation due to Unrecognized token 'new': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false') at [Source: (String)"{ "time_created": { "$gte": new Date(new Date().setHours(0, 0, 0, 0)), "$lt": new Date(new Date().setHours(23, 59, 59, 999)) } }"; line: 3, column: 16]
I tried converting to JSON format but it doesn't work:
{
"time_created": {
"$gte": { "$date": "${now():format('yyyy-MM-dd\'T\'HH:mm:ss.SSS\'Z\'')}" },
"$lt": { "$date": "${now():format('yyyy-MM-dd\'T\'23:59:59.999\'Z\'')}" }
}
}
and
{
"time_created": {
"$gte": "${now():toDate():format('yyyy-MM-dd\'T\'HH:mm:ss.SSS\'Z\'')}",
}
}
but it not work.
Please help me, thanks.
Can you help me convert mongodb query to json format working with GetMongo ?
{
"time_created": {
"$gte": new Date(new Date().setHours(0, 0, 0, 0)),
"$lt": new Date(new Date().setHours(23, 59, 59, 999))
}
}
Give me another more suitable solution ?
Please help me, thanks.

I fixed it. Nifi's getMongo Query field doesnt support EL. So i created a stored function in MongoDB for my dynamic query and called it from Nifi.
Create function in Mongodb:
db.system.js.insertOne({
_id: "find_data_by_date_range",
value : function(created_time) {
var fromDate = new Date(new Date().setHours(0, 0, 0, 0));
var toDate = new Date(new Date().setHours(23, 59, 59, 999));
if ((created_time >= fromDate) && (created_time <= toDate)) {
return true;
}
return false;
}
});
Query Nifi:
{"$where": "find_data_by_date_range(this.time_created)"}
It is worked!

Related

$matching with field added with $dateToString doesn't work

In my MongoDB collection I have documents that contain a nested string field, containing a month and year, e.g. '04/2021'. Sample document:
{
"_id": {
"$oid": "608ba45cec43c5b24cda034b"
},
"status": "pass",
"stage": 5,
"priority": 0,
"payload": {
"company_id": "8800",
"company_name": "<MY COMPANY>",
"target_period": "04/2021"
},
"retry_count": 0,
"build_number": "101",
"job_name": "P123",
"createdAt": {
"$date": "2021-04-30T06:31:56.000Z"
},
"updatedAt": {
"$date": "2021-05-10T03:55:44.686Z"
}
}
I am trying to write an aggregation pipeline that will dynamically return documents where said field points to the past month. For example, ran this month (May 2021) I would get documents labeled with '04/2021'. From this post I found the oneliner for getting the comparison string: new Date(new Date().getFullYear(), new Date().getMonth(), 1). (I understand that by the virtue of getMonth returning a zero-based index of month, getting the previous month works by accident and has to be solved somehow.)
This pipeline does not work:
[
{
$addFields: {
previous_month: {
$dateToString: {
'date': new Date(new Date().getFullYear(), new Date().getMonth(), 1),
'format': '%m/%G'
}
}
}
},
{
$match: {
"payload.target_period": "$previous_month"
}
}
]
With MongoDB Compass I can see that the field previous_month is populated just fine by the $addFields stage (above sample document gets value 04/2021), but the $match stage returns 0 documents. I'm running MongoDB version 4.2.12.
You should use $expr operator while trying to self reference another ket in a document inside
$match stage.
[
{
$addFields: {
previous_month: {
$dateToString: {
'date': new Date(new Date().getFullYear(), new Date().getMonth(), 1),
'format': '%m/%G'
}
}
}
},
{
$match: {
$expr: {
$eq: [ "$payload.target_period", "$previous_month" ],
},
}
}
]
Instead of doing whole process in query, I think you can prepare input date in your client language, (js, nodejs) easily,
have prepared a function zeroFill it will return number with concat 0 if its less than 10,
get previous month date and pick previous month
concat both month and year
function zeroFill(i) { return (i < 10 ? '0' : '') + i; }
var date = new Date();
date.setMonth(date.getMonth() - 1);
let searchDate = zeroFill(date.getMonth() + 1) + "/" + date.getFullYear();
console.log(searchDate); // mm/yyyy
Your query would be just:
[{ $match: { "payload.target_period": searchDate } }]
Playground
I would suggest moment.js library, it is much simpler to use:
{ $match: { "payload.target_period": moment().startOf("months").subtract(1, "months").format("MM/YYYY") } }

mongodb sum aggregation return float instead of integer

I have a document which history embedded field is as below:
first_pn_data = {'_cls': 'InstallationEmbeddedHistory',
'coverage': 0.8197712971965834,
'date': datetime.datetime(2020, 9, 7, 0, 0),
'estimated_installs': 39349022,
'popularity_rank': 0,
'rank': 1851}
second_pn_data = {'_cls': 'InstallationEmbeddedHistory',
'coverage': 0.8197712971965834,
'date': datetime.datetime(2020, 9, 7, 0, 0),
'estimated_installs': 23412618,
'popularity_rank': 0,}
The estimated_installs field is an integer.
when I run the following query:
query = [
{
"$match": {
"package_name": {
"$in": ["first_pn", "second_pn"]
}
}
},
{
"$unwind": "$history"
},
{
"$group": {
"_id": "$history.date",
"total": {
"$sum": "$history.estimated_installs"
}
}
}
]
the result for the above date is:
{'_id': datetime.datetime(2020, 9, 7, 0, 0), 'total': 62761640.54968266}
while I expect the total=62761640.
I don't know why the result is float and has a decimal.
Can someone help me with this?

deleting records mongo db in collection

How can I use below script for deleting 5K records:
db.transaction.deleteMany({
"timeStamp": { $gte: new Date(2018, 0, 1), $lt: new Date(2019, 0, 1) }
})
I tried with finding first in collection, but it's saying 0 records fetched.
db.transaction.find({
"timeStamp": { $gte: new Date(2018, 0, 1), $lt: new Date(2019, 0, 1) }
})
Output: Fetched 0 record(s) in 31ms
But i checked in collection below records with these timestamp are present,
"timeStamp" : ISODate("2018-12-31T18:30:03.379Z"),
"timeStamp" : ISODate("2018-12-31T18:30:03.982Z"),

Mongodb get lastHour for each day

I have the following schema in mongodb, where the timestamp is the timestamp at an hourly level
{
"day_chan1" : 54.464,
"day_chan2" : 44.141,
"day_chan3" : 44.89,
"gatewayId" : "443719005AA3",
"timestamp" : ISODate("2016-02-15T23:00:00.000Z"),
"total_curr_chan" : 5.408,
"total_day_chan" : 143.495,
"type" : 1
}
I want to be able to query the last timestamp for the day for the last 7 days and 30 days. In order to do this, I am thinking of doing something like
var d = new Date(); // today!
for(var i =1; i <= 7; i++) {
var n = i; // go back n days!
d.setDate(d.getDate() - n);
d.setHours(23,0,0);
var query = {
gatewayId: req.params.deviceId,
timestamp: { $lt: new Date(d) }
};
db
.find(query,function(resp) {
//return the data here
});
}
But this creates a problem of multiple callbacks and I want to know if there is an easier way of doing so using aggregates or some other method
Use the $hour operator within the $project operator to extract the hour part of the timestamp, then query with $match to filter documents that do not satisfy the given hour criteria:
var pipeline = [
{
"$project": {
"day_chan1": 1,
"day_chan2": 1,
"day_chan3": 1,
"gatewayId": 1,
"timestamp": 1,
"total_curr_chan": 1,
"total_day_chan": 1,
"type": 1,
"hour": { "$hour": "$timestamp" }
}
},
{ "$match": { "hour": 23 } }
];
collection.aggregate(pipeline, function(err, result) {
//return the data here
console.log(result);
});
For arbitrary last hour it must be a bit more complex:
db.collection.aggregate([
{$match:{
timestamp:{$type: "date"}}
// add date constraints here
},
{$project:{
_id:1,
date:{"y":{$year:"$timestamp"}, "d":{$dayOfYear:"$timestamp"}},
doc:"$$CURRENT"}
},
{$group:{
_id:"$date",
maxtime: {$max:"$doc.timestamp"},
doc:{$push:"$doc"}}
},
{$unwind:"$doc"},
{$project:{
latest: {$cmp: ["$maxtime", "$doc.timestamp"]},
doc:"$doc"}
},
{$match:{"latest":0}}
])
With map-reduce it should be simpler, but may be slower.

MongoDB find Query comparision with CurrentDate

I want to fetch current day documents from a MongoDB collection. My documents look like this:
{
"_id" : ObjectId("55743941789a9abe7f4af3fd"),
"msisdn" : "9xxxxxxxxxx",
"act_date" : ISODate("2014-11-24T00:00:00Z"),
"date" : ISODate("2015-06-07T00:00:00Z"),
"recharge" : { "recharge_amt" : 0, "rechargetype" : "WEB" },
"voice" : { "local_og_mou" : 20, "local_other_mobile_og_mou" : 0, "nld_og_mou" : 0, "nld_other_mobile_og_mou" : 10 },
"gprs" : { "total_access_count" : 1, "total_datavolume_mb" : 42 },
"sms" : { "freesms" : 3, "local_sms_count" : 0, "nat_sms_count" : 0, "inter_sms_count" : 0 }
}
As per your question you want to fetch current day documents from a mongodb collection. In mongoDB shell when you type new Date() it gives you current date with time and this value always vary when you run same new Date() so probably your query like this :
db.collectionName.find({"start_date":new Date()}).pretty()
But, I think this query return the those documents which will presents in your collection but same current Date value may be not presents in your documents SO this case you should use following
db.collectionName.find({"start_date":{"$lte":new Date()}}).pretty()
Or
db.collectionName.find({"start_date":{"$gte":new Date()}}).pretty()
In some case If you want to find exact match with year,month,day then you should use aggregation with $year,$month,$dayOfMonth in $project like this :
db.collectionName.aggregate({
"$project": {
"year": {
"$year": "$date"
},
"month": {
"$month": "$date"
},
"day": {
"$dayOfMonth": "$date"
}
}
}, {
"$match": {
"year": new Date().getFullYear(),
"month": new Date().getMonth() + 1, //because January starts with 0
"day": new Date().getDate()
}
})
In above aggregation query will return those documents which match current date like year,month,day of current date. Also you replace $match as
var currentDate = new Date()
{
"$match": {
"year": currentDate.getFullYear(),
"month": currentDate.getMonth() + 1, //because January starts with 0
"day": currentDate.getDate()
}
}
If you are using Mongoose you can include timeStamp: true after your schema definition to get autogenerated createdAt and updatedAt fields, Mongoose will take care of it.
const itemSchema = mongoose.Schema({
// Your Schema definition here
}, {
timestamps: true
})
In your case, you need to compare your TimeStamp key with the start of today which is 12 AM with ISO string as 2019-11-08T00:00:00.000Z and end of the day which is 11:59 PM with ISO string as 2019-11-08T23:59:59.999Z.
The below code will do it for you.
let queryObj = {}
const startOfDay = new Date(new Date().setUTCHours(0, 0, 0, 0)).toISOString()
const endOfDay = new Date(new Date().setUTCHours(23, 59, 59, 999)).toISOString()
queryObj.createdAt = {
$gte: startOfDay, // 2019-11-08T00:00:00.000Z
$lt: endOfDay // 2019-11-08T23:59:59.999Z
}
let items = item.find(obj)
// new Date().setUTCHours(0, 0, 0, 0) Generates this weird string '1573171200000' which is not a human readable format of TimeStamp
// To convert it into ISO String we have .toISOString() method
You can do this using below query.
db.collection.find({"start_date" : { $gte : new ISODate("2015-05-27T00:00:00Z") }});
Note : above query will return documents which are greather than specified date.
To find a date that equals another date
db.collection.find({"start_date" : new ISODate("2015-05-27T00:00:00Z") });