I have this mongoose schema, it stores user's salary
const salarySchema = new mongoose.Schema(
{
userId: {
type: mongoose.Schema.Types.ObjectId,
ref: 'User',
required: true,
trim: true,
},
amount: {
type: String,
required: true,
trim: true,
},
paymentType: {
type: String,
required: true,
enum: [PAYMENT_TYPE],
default: PAYMENT_TYPE.MONTHLY,
},
startDate: {
type: Date,
required: true,
},
endDate: {
type: Date,
required: false,
default: null,
},
}
);
If I have a time frame given I want to query all salaries applicable to that time frame using startDate and endDate attributes.
This is the graphical representation of all possible combinations, The given time frame is April to March,
APRIL MARCH
| —------------FILTER--------------- |
| —-------- |
| —-------------------------------------------------- |
| —--------------------- |
| —------------------------- |
| —----- |
| —------- |
The final two should not be included in the result because it's not applicable to the given time frame.
And the there can be salary the endDate can be null and it represent the user active salary,
Basicaly I am storing the salary history also using start,end date.
{
$or: [
// DB record range is completely within provided range
{
startDate: { $gte: start, $lte: end },
endDate: { $gte: start, $lte: end },
},
// Provided range is completely within DB record range
{
startDate: { $lte: start },
$or: [{ endDate: { $gte: end } }, { endDate: null }],
},
// Left intersection
{
startDate: { $lte: start },
endDate: { $lte: end, $gte: start },
},
// Right intersection
{
startDate: { $gte: start, $lte: end },
$or: [{ endDate: { $gte: end } }, { endDate: null }],
},
],
},
Related
I created a web-scraper to store data for a week to find a trend.
I wrote code to delete data from more than week ago every time the script runs.
However the data is still being stored for more than a week ago, is there a reason for this?"
example coin data createdAt field looks like
"createdAt": {
"$date": "2021-08-11T10:55:19.843Z"
},
coinSchema.statics.deleteOldData = async function () {
// delete old data
const today = new Date(Date.now());
today.setHours(0, 0, 0, 0);
const oneWeekAgo = new Date(Date.now());
const pastDate = oneWeekAgo.getDate() - 7;
oneWeekAgo.setDate(pastDate);
await this.deleteMany({
createdAt: {
$gte: today,
}, // 16 < 17 wont delete it prevent duplicates for one day
});
await this.deleteMany({
createdAt: {
$lt: pastDate,
}, // from 1 week ago
});
};
in the script i have the this run
async function main() {
await Coin.deleteOldData();
my coin model looks like :
const coinSchema = mongoose.Schema(
{
specNo: {
type: String,
required: true,
},
coinName: {
type: String,
required: true,
},
fullName: {
type: String,
required: false,
},
category: {
type: String,
},
array: [
{
GradeName: String,
PopulationCount: String,
trend: { type: Number, default: 0 },
},
],
},
{
timestamps: true,
}
);
Did you look at TTL based index? https://docs.mongodb.com/manual/core/index-ttl/
This is a good way to cleanup old data where DB itself takes care of it. In your case 7 days is 604800 seconds so if you create a index on createdAt with ttl 604800 then you should be all good!
db.collection.createIndex( { "createdAt ": 1 }, { expireAfterSeconds: 604800 } )
You can do it like this:
db.collection.deleteMany({
createdAt: { $lte: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString() },
})
First of all, I have this type of model:
const produkSchema = new mongoose.Schema({
nama_produk: String,
etalase: {type: mongoose.Schema.Types.ObjectID, ref: 'kategori'},
kategori: {type: mongoose.Schema.Types.ObjectID, ref: 'kategori'},
jenis: {type: mongoose.Schema.Types.ObjectID, ref: 'kategori.jenis'},
bahan: String,
warna: String,
deskripsi: String,
foto_produk: [String],
harga: Number,
link_bukalapak: String,
link_shopee: String,
link_tokopedia: String,
link_lazada: String,
link_website: String,
display: {type: Boolean, default: false},
}, {
weights: {
nama_produk: 5,
},
timestamps: true
})
const tokoSchema = new mongoose.Schema({
username: {type: String, trim: true},
password: {type: String, required: true, select: false},
merek: String,
listMerek: [{type: mongoose.Schema.Types.ObjectID, ref: 'produk'}],
deskripsi: String,
follower: [{type: mongoose.Schema.Types.ObjectID, ref: 'user'}],
email: {type: String, trim: true, unique: true},
instagram: String,
whatsapp: String,
website: String,
alamat: String,
foto_profil: String,
bukalapak: String,
shopee: String,
tokopedia: String,
fotoktp: String,
banner: [{
gambar: {type: String, required: true, trim: true},
// order: {type: Number, required: true},
}],
produk: [produkSchema],
etalase: [{type: mongoose.Schema.Types.ObjectID, ref: 'kategori'}],
// etalase: [{
// kategori: {type: mongoose.Schema.Types.ObjectID, ref: 'kategori'},
// order: {Number}
// }],
approve: {type: Number, default: 0}, // 0: pending, 1: reject, 2: approve
populer: {type: Boolean, default: false},
gambar_populer: [String],
pilihan: {type: Boolean, default: false},
}, {timestamps: true});
and I have an endpoint to filter this produkSchema with following code:
exports.filterProduk = (req, res) => {
const {merek, warna, kategori, jenis, hargaAwal, hargaAkhir, skip, limit} = req.body
let query = {}
const $and = []
if (merek) {
$and.push({$or: merek.map(id => ({_id: mongoose.Types.ObjectId(id)}))})
}
if (warna) {
$and.push({$or: warna.map(warna => ({"produk.warna": warna}))})
}
if (kategori) {
query["produk.etalase"] = mongoose.Types.ObjectId(kategori)
}
if (jenis) {
$and.push({$or: jenis.map(id => ({"produk.jenis": mongoose.Types.ObjectId(id)}))})
}
if (hargaAwal !== '') {
query["produk.harga"] = {
$gte: parseInt(hargaAwal),
}
}
if (hargaAkhir !== '') {
query["produk.harga"] = {
$lte: parseInt(hargaAkhir)
}
}
if ($and.length > 0) {
query = {$and, ...query}
}
toko.aggregate([
{$unwind: '$produk'},
{$match: query},
{
$lookup: {
from: "kategoris",
as: "produk.etalase",
let: {pjid: "$produk.jenis"},
pipeline: [
{$unwind: "$jenis"},
{$match: {$expr: {$eq: ["$$pjid", "$jenis._id"]}}},
{
$project: {
"jenis._id": 1,
"jenis.label": 1
}
}
]
}
},
{$unwind: {path: "$produk.etalase"}},
{$group: {_id: '$_id', produk: {$push: '$produk'}, foto_profil: {$first: '$foto_profil'}}},
{$limit: skip + limit},
{$skip: skip}
])
.then(async data => {
res.status(200).json({data, prefix: {produk: "uploads/produk", toko: "uploads/toko"}})
})
}
the actual result is:
[
{_id: blabla,
foto_profil: "blabla",
produk:[{nama_produk: "blabla", bahan: "blabla", ...rest ProdukSchema as query}]
},
{_id: blabla,
foto_profil: "blabla",
produk:[{nama_produk: "blabla", bahan: "blabla", ...rest ProdukSchema as query}]
},
{_id: blabla,
foto_profil: "blabla",
produk:[{nama_produk: "blabla", bahan: "blabla", ...rest ProdukSchema as query}]
}
]
expected:
[
{nama_produk: "blabla", bahan: "blabla", ...rest ProdukSchema as query, foto_profil(from toko schema): "blabla"},
{nama_produk: "blabla", bahan: "blabla", ...rest ProdukSchema as query, foto_profil(from toko schema): "blabla"}
]
and having pagination with this produkSchema (limit and offset)
actually, before this, I already ask for this query at here, but this query will produce a lot of data and need to be paginated,
how do I do this? Should I split my produkSchema to main subdocument? or any query exist for this condition?
I am not sure with exact requirement, but I can guess 2 options,
Assuming Variables:
let skip = 0;
let limit = 10;
First Option:
remove $group stage from the end and start after $unwind stage
$replaceRoot to merge objects produk and foto_profil using $mergeObjects and replace the root
{
$replaceRoot: {
newRoot: {
$mergeObjects: [{ foto_profil: "$foto_profil" }, "$produk"]
}
}
},
{ $skip: skip * limit },
{ $limit: limit }
Playground
Second Option:
remove $group stage from the end and start after $unwind stage
$group by toko id and produk id, this will group produk and get unique first produk
$replaceRoot to merge objects produk and foto_profil using $mergeObjects and replace the root
{
$group: {
_id: {
_id: "$_id",
produk_id: "$produk._id"
},
root: { $first: "$$ROOT" }
}
},
{
$replaceRoot: {
newRoot: {
$mergeObjects: [{ foto_profil: "$root.foto_profil" }, "$root.produk"]
}
}
},
{ $skip: skip * limit },
{ $limit: limit }
Playground
maybe you can do this after lookup:
$unwind produk (where the produk still on array type, then unwind it).
$group: { _id: nama_produk, etc... }
$project ...... or whatever you wanna do with.
so you can get the list of group by nama_produk as an _id.
My code on the backend, in case it matters (NodeJS and MogoDB):
//my includes at the top of the file
const mongoose = require('mongoose');
const Appt = mongoose.model('Appt');
const ApptType = mongoose.model('ApptType');
const ApptStatus = mongoose.model('ApptStatus');
var moment = require('moment-timezone');
moment().tz('America/New_York');
now = moment(); // add this 2 of 4
dayStart = now.startOf('day');
dayEnd = now.endOf('day');
// the aggregation query that's not returning correctly
Appt.aggregate([
{
$match: {
patientID: appt.patientID._id,
scheduled: {
$gte: new Date(start),
$lt: new Date(appt.pmtduedate)
}
}
},
{
$group: {
_id: 'id',
payment: { $sum: '$payment' },
pmtdue: { $sum: '$pmtdue' },
visits: { $sum: 1 }
}
}
]).exec(
err => {
console.log(`Error finding past payments`, err);
callback(err);
},
result => {
console.log(`RESULT: ${result}`);
pastPayments = result;
if (!pastPayments || pastdueamt === 0) {
pastdueamt = 0;
console.log(`2. getCurrentDue ${pastdueamt}`);
this.getCurrentDue(appt, pastdueamt, today, callback);
} else {
console.log(`pastPayments ${pastPayments}`);
console.log(
`planamt ${planamt} pmtdue ${pastPayments.pmtdue} payments: ${pastPayments.payment}`
);
pastdueamt =
pastPayments.pmtdue === 0
? planamt - pastPayments.payment
: pastPayments.pmtdue - pastPayments.payment;
console.log(`pastdueamt calculated: ${pastdueamt}`);
console.log(`2. getCurrentDue`);
this.getCurrentDue(appt, pastdueamt, today, callback);
}
}
);
When I run my query in mongo, the expected results return. In my app, the results of this query above return nothing (no error, either). I've tried doing the following:
$match: {
patientID: new mongoose.types.ObjectId(appt.patientID._id),
I've also tried:
$match: {
patientID: { $toObjectId: appt.patientID._id },
but I get errors on both of these options. The first returns an error of
TypeError: Cannot read property 'ObjectId' of undefined.
The second returns some sort of mongo error
errmsg: 'unknown operator: $toObjectId',
code: 2,
codeName: 'BadValue',
name: 'MongoError',
[Symbol(mongoErrorContextSymbol)]: {} }
How do I do mongoose aggregation successfully using objectIds, virtual types, etc.?
EDITED TO ADD MY SCHEMAS:
const apptSchema = new mongoose.Schema(
{
ID: Number,
patientID: {
type: mongoose.Schema.Types.ObjectId,
ref: 'Patient'
},
oldPatientID: Number,
status: {
type: mongoose.Schema.Types.ObjectId,
ref: 'ApptStatus'
},
type: {
type: mongoose.Schema.Types.ObjectId,
ref: 'ApptType'
},
scheduled: Date,
note: String,
reminder: Boolean,
cell: Boolean,
email: Boolean,
subjective: String,
assessment: String,
plan: String,
planamt: Number,
objective: {
clearUC: Boolean,
UCcheck: String,
thompson: String,
activator: String,
other: String
},
updated: {
type: Date,
default: new Date()
},
pmtdue: Number,
pmtduedate: Date,
payment: Number,
pmttype: String,
paid: Boolean,
pmtnote: String
},
{ toJSON: { virtuals: true } }
);
I have these collections, stats and items. Stats has one item as subdocument:
var ItemSchema = new Schema({
type: String,
created_at: Date,
updated_at: {
type: Date,
default: Date.now()
}
});
var StatsSchema = new Schema({
item: {
type: Schema.ObjectId,
ref: 'Item'
},
url: String,
date: Date,
action: String,
hourly: Number
});
I'd like to aggregate Stats grouping by item.type. Is it possible?
I tried something like this but without luck:
db.stats.aggregate(
{ $project: { _id: 1, hourly: 1, action: 1, item: 1, type: '$item.type' } },
{ $group: { _id: { action: '$action', type: '$item.type' }, total: { $sum: '$hourly' } } }
)
You should not need the $project part of the pipeline.
You should be get what you need from the $group stage
db.stats.aggregate({ $group: { _id: "$item.type" , total: { $sum: '$hourly' } } });
I have the following documents
User Schema:
var UserSchema = new Schema({
name: String,
email: { type: String, lowercase: true },
offers: [],
anyCountry: {type: Boolean, default: false},
city: String,
});
Tags Schema
var TagSchema = new Schema({
text: String,
dateCreated: { type: Date, default: Date.now}
});
I am aggregating it this way:
User.aggregate(
{$match: {
$or: [
{'isBlocked': false},
{'isBlocked': {$exists: false}}
]}},
{ $project: {"offers": 1, _id: 0, city: 1, name: 1}},
{ $unwind: "$offers" },
{
$match: {
$and: [
{'offers': { $not: { $size: 0} }},
{'offers.type': type}
]
}
},
{ $sort: {"offers.dateCreated": -1} },
function (err, result) {
if (!err) {
return res.json({status: 'success', data: result});
} else {
return res.send(err);
}
}
)
The output is ok, but it contains tags as array. What I need is:
to have array values assigned to a computed field "offers.tagsList" as a coma separated sting {offers.tagsList = 'tag1, tag2, tag3, ...'}.
check if filed offers.anyCountry doesn't exists and add it to the output with value false.
Thanks!