my need is copy one subdocument from a document and insert into the
same document.But i use forEach and findAndModify it can not insert
into the same document.
document example:
{
"_id" : ObjectId("59b5e84d71ab5580d643d070"),
"modifiedOn" : ISODate("2019-04-03T14:57:22.177+0000"),
"modifiedBy" : "XXX",
"createdOn" : ISODate("2017-09-09T16:33:34.464+0000"),
"createdBy" : "liuyu",
"channelSales" : [
{
"platform" : "amazon",
"channel" : "amazon_ca",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dc1")
},
{
"platform" : "amazon",
"channel" : "amazon_uk",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dc0")
},
{
"platform" : "amazon",
"channel" : "amazon_us",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dbf")
},
{
"platform" : "amazon",
"channel" : "amazon_jp",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dbe")
},
{
"platform" : "amazon",
"channel" : "amazon_de",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dbd")
},
{
"platform" : "amazon",
"channel" : "amazon_es",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dbc")
},
{
"platform" : "amazon",
"channel" : "amazon_fr",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dbb")
},
{
"platform" : "amazon",
"channel" : "amazon_it",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dba")
},
{
"platform" : "ebay",
"channel" : "ebay_au",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db9")
},
{
"platform" : "ebay",
"channel" : "ebay_de",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db8")
},
{
"platform" : "ebay",
"channel" : "ebay_es",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db7")
},
{
"platform" : "ebay",
"channel" : "ebay_fr",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db6")
},
{
"platform" : "ebay",
"channel" : "ebay_it",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db5")
},
{
"platform" : "ebay",
"channel" : "ebay_uk",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db4")
},
{
"platform" : "ebay",
"channel" : "ebay_us",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db3")
},
{
"platform" : "walmart",
"channel" : "walmart_us",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5a4d9de2bb1aee844f03e1a6")
},
{
"platform" : "walmart",
"channel" : "walmart_ca",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5a4d9de2bb1aee844f03e1a5")
},
{
"platform" : "amazon",
"channel" : "amazon_au",
"saleStatus" : "T",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5abe095bb1d48d194f6187c0")
},
{
"platform" : "amazon",
"channel" : "amazon_in",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5c9af2776f3dcf04491818f2")
}
],
"statusLevel" : "",
"statusType" : "",
"status" : "A",
"skuId" : "abc001",
"__v" : NumberInt(3)
}
i want to copy:
{
"platform" : "walmart",
"channel" : "walmart_us",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5a4d9de2bb1aee844f03e1a6")
}
and change the "channel" : "walmart_dsv",the other files is the same like this :
{
"platform" : "walmart",
"channel" : "walmart_dsv",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5a4d9de2bb1aee844f03e1a6")
}
and insert into the same document.
i use this command:
db.getCollection("0521").aggregate([
{$unwind: "$channelSales"},
{$project: {platform: "$channelSales.platform",
channel: "$channelSales.channel",
saleStatus: "$channelSales.saleStatus",
type: "$channelSales.type",
url: "$channelSales.url",
remark: "$channelSales.remark",
isCaught: "$channelSales.isCaught",
_id: "$channelSales._id"
}},
{ $match : { "channel" : "amazon_us"} }
]).forEach(function(award_team){
if(award_team != null)
{
db.getCollection("0521").findAndModify(
{
query: {_id: award_team._id},
update: { $push: {channelSales: [ {platform: award_team.platform, channel: "walmart_dsv", saleStatus: award_team.saleStatus, type: award_team.type, url: award_team.url, remark: award_team.remark, isCaught: award_team.isCaught, _id: award_team._id }] } },
upsert: true,
});
}
});
but it is add a new document,what can i do?
You are making a mistake in projection:
You are using sub-document _id to update the main document and you are using option {upsert: true} so every time it is inserting a new document in case it does not find _id.
Update your project as below:
db.collection.aggregate([
{$unwind: "$channelSales"},
{$project: {
_id: "$_id", // Main Document Id
platform: "$channelSales.platform",
channel: "$channelSales.channel",
saleStatus: "$channelSales.saleStatus",
type: "$channelSales.type",
url: "$channelSales.url",
remark: "$channelSales.remark",
isCaught: "$channelSales.isCaught",
channelSalesId: "$channelSales._id" // Sub-Document Id (Caannel Sales Id)
}},
{ $match : { "channel" : "amazon_us"} }
])
Now you will get a response like below:
{
"_id" : ObjectId("59b5e84d71ab5580d643d070"),
"platform" : "amazon",
"channel" : "amazon_us",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"channelSalesId" : ObjectId("59b5f86aaa0ee15555a39dbf")
}
Now you can use "award_team._id" to update Document:
db.getCollection("0521").findAndModify(
{
query: {_id: award_team._id},
update: { $push: {channelSales: [ {platform: award_team.platform, channel: "walmart_dsv", saleStatus: award_team.saleStatus, type: award_team.type, url: award_team.url, remark: award_team.remark, isCaught: award_team.isCaught, _id: award_team._id }] } },
upsert: true,
});
}
});
Related
I'm new to mongodb and in this question I have 2 collections, one is selected_date, another is global_mobility_report, what I'm trying to do is to find entries in global_mobility_report whose date is in the selected_date so I use $lookup to join the two collections.
date_selected:
{
"_id" : ObjectId("5f60d81ba43174cf172ebfdc"),
"date" : ISODate("2020-05-22T00:00:00.000+08:00")
},
{
"_id" : ObjectId("5f60d81ba43174cf172ebfdd"),
"date" : ISODate("2020-05-23T00:00:00.000+08:00")
},
{
"_id" : ObjectId("5f60d81ba43174cf172ebfde"),
"date" : ISODate("2020-05-24T00:00:00.000+08:00")
},
{
"_id" : ObjectId("5f60d81ba43174cf172ebfdf"),
"date" : ISODate("2020-05-25T00:00:00.000+08:00")
},
{
"_id" : ObjectId("5f60d81ba43174cf172ebfe0"),
"date" : ISODate("2020-05-26T00:00:00.000+08:00")
},
{
"_id" : ObjectId("5f60d81ba43174cf172ebfe1"),
"date" : ISODate("2020-05-27T00:00:00.000+08:00")
}
global_mobility_report:
{
"_id" : ObjectId("5f49fb013acddb5eec37f99e"),
"country_region_code" : "AE",
"country_region" : "United Arab Emirates",
"sub_region_1" : "",
"sub_region_2" : "",
"metro_area" : "",
"iso_3166_2_code" : "",
"census_fips_code" : "",
"date" : "2020-02-15",
"retail_and_recreation_percent_change_from_baseline" : "0",
"grocery_and_pharmacy_percent_change_from_baseline" : "4",
"parks_percent_change_from_baseline" : "5",
"transit_stations_percent_change_from_baseline" : "0",
"workplaces_percent_change_from_baseline" : "2",
"residential_percent_change_from_baseline" : "1"
},
{
"_id" : ObjectId("5f49fb013acddb5eec37f99f"),
"country_region_code" : "AE",
"country_region" : "United Arab Emirates",
"sub_region_1" : "",
"sub_region_2" : "",
"metro_area" : "",
"iso_3166_2_code" : "",
"census_fips_code" : "",
"date" : "2020-02-16",
"retail_and_recreation_percent_change_from_baseline" : "1",
"grocery_and_pharmacy_percent_change_from_baseline" : "4",
"parks_percent_change_from_baseline" : "4",
"transit_stations_percent_change_from_baseline" : "1",
"workplaces_percent_change_from_baseline" : "2",
"residential_percent_change_from_baseline" : "1"
},
{
"_id" : ObjectId("5f49fb013acddb5eec37f9a0"),
"country_region_code" : "AE",
"country_region" : "United Arab Emirates",
"sub_region_1" : "",
"sub_region_2" : "",
"metro_area" : "",
"iso_3166_2_code" : "",
"census_fips_code" : "",
"date" : "2020-02-17",
"retail_and_recreation_percent_change_from_baseline" : "-1",
"grocery_and_pharmacy_percent_change_from_baseline" : "1",
"parks_percent_change_from_baseline" : "5",
"transit_stations_percent_change_from_baseline" : "1",
"workplaces_percent_change_from_baseline" : "2",
"residential_percent_change_from_baseline" : "1"
},
{
"_id" : ObjectId("5f49fb013acddb5eec37f9a1"),
"country_region_code" : "AE",
"country_region" : "United Arab Emirates",
"sub_region_1" : "",
"sub_region_2" : "",
"metro_area" : "",
"iso_3166_2_code" : "",
"census_fips_code" : "",
"date" : "2020-02-18",
"retail_and_recreation_percent_change_from_baseline" : "-2",
"grocery_and_pharmacy_percent_change_from_baseline" : "1",
"parks_percent_change_from_baseline" : "5",
"transit_stations_percent_change_from_baseline" : "0",
"workplaces_percent_change_from_baseline" : "2",
"residential_percent_change_from_baseline" : "1"
}
when I try to find all entries in global with 'date' match in selected_date(I have converted the string to data format in gobal_mobility_report), it returns empty array.
db.global_mobility_report.aggregate([
{$match:{country_region:"Indonesia"}},
{$addFields: {"dateconverted": {$convert: { input: "$date", to: "date", onError:"onErrorExpr", onNull:"onNullExpr"}:}}},
{
$lookup:
{
from: "selected_date",
localField:"dateconverted",
foreignField: "date",
as: "selected_dates" // empty
}
})]
The output is:
{
"_id" : ObjectId("5f49fd6a3acddb5eec4427bb"),
"country_region_code" : "ID",
"country_region" : "Indonesia",
"sub_region_1" : "",
"sub_region_2" : "",
"metro_area" : "",
"iso_3166_2_code" : "",
"census_fips_code" : "",
"date" : "2020-02-15",
"retail_and_recreation_percent_change_from_baseline" : "-2",
"grocery_and_pharmacy_percent_change_from_baseline" : "-2",
"parks_percent_change_from_baseline" : "-8",
"transit_stations_percent_change_from_baseline" : "1",
"workplaces_percent_change_from_baseline" : "5",
"residential_percent_change_from_baseline" : "1",
"dateconverted" : ISODate("2020-02-15T08:00:00.000+08:00"),
"selected_dates" : [ ]
},
{
"_id" : ObjectId("5f49fd6a3acddb5eec4427bc"),
"country_region_code" : "ID",
"country_region" : "Indonesia",
"sub_region_1" : "",
"sub_region_2" : "",
"metro_area" : "",
"iso_3166_2_code" : "",
"census_fips_code" : "",
"date" : "2020-02-16",
"retail_and_recreation_percent_change_from_baseline" : "-3",
"grocery_and_pharmacy_percent_change_from_baseline" : "-3",
"parks_percent_change_from_baseline" : "-7",
"transit_stations_percent_change_from_baseline" : "-4",
"workplaces_percent_change_from_baseline" : "2",
"residential_percent_change_from_baseline" : "2",
"dateconverted" : ISODate("2020-02-16T08:00:00.000+08:00"),
"selected_dates" : [ ]
}
The reason you are getting an empty array is because dateconverted does not match the date field.
The $lookup operator does an equality between the localField and the foreigntField field, so basically with an example
db.users.insertMany([
{ email: "test#example.com", userId: 0 },
{ email: "test2#example.com", userId: 1 },
{ email: "test3#example.com", userId: 2 },
{ email: "test3#example.com", userId: 3 }
]);
db.posts.insertMany([
{ by: 0, post: "hello world" },
{ by: 0 , post: "hello earthlings" },
{ by: 3, post: "test test test"}
]);
db.posts.aggregate([
{
$lookup: {
from: "users",
localField: "by",
foreignField: "userId",
as: "list_of_post"
}
}
]).toArray();
The output will be what it suppose to be, because the localField matched the ForeignField
[
{
"_id" : ObjectId("5f60f6859a6df3133b325eb0"),
"by" : 0,
"post" : "hello world",
"list_of_post" : [
{
"_id" : ObjectId("5f60f6849a6df3133b325eac"),
"email" : "test#example.com",
"userId" : 0
}
]
},
{
"_id" : ObjectId("5f60f6859a6df3133b325eb1"),
"by" : 0,
"post" : "hello earthlings",
"list_of_post" : [
{
"_id" : ObjectId("5f60f6849a6df3133b325eac"),
"email" : "test#example.com",
"userId" : 0
}
]
},
{
"_id" : ObjectId("5f60f6859a6df3133b325eb2"),
"by" : 3,
"post" : "test test test",
"list_of_post" : [
{
"_id" : ObjectId("5f60f6849a6df3133b325eaf"),
"email" : "test3#example.com",
"userId" : 3
}
]
}
]
Let's mimic a situation where it does not match
db.posts.drop();
db.posts.insertMany([
{ by: 20, post: "hello world" },
{ by: 23 , post: "hello earthlings" },
{ by: 50, post: "test test test"}
]);
We get an empty array
[
{
"_id" : ObjectId("5f60f83344304796ae700b4d"),
"by" : 20,
"post" : "hello world",
"list_of_post" : [ ]
},
{
"_id" : ObjectId("5f60f83344304796ae700b4e"),
"by" : 23,
"post" : "hello earthlings",
"list_of_post" : [ ]
},
{
"_id" : ObjectId("5f60f83344304796ae700b4f"),
"by" : 50,
"post" : "test test test",
"list_of_post" : [ ]
}
]
So, back to your question, the reason for the empty array is as a result of the dateconverted field not matching the date field. So, let's take a look at an example.
In the first document the dateconverted is
ISODate("2020-02-16T08:00:00.000+08:00") and checking at date_selected document , there is no field that correspond to this value ISODate("2020-02-16T08:00:00.000+08:00"). But let's manually insert this, so you will properly understand what I am talking about.
db.date_selected.insert({
"_id" : ObjectId(),
"date": ISODate("2020-02-16T08:00:00.000+08:00")
});
Running the aggregation pipeline will also make selected_dates an empty array. And the other thing you have to note is that the mm/dd/yyy part of the ISODate object does not also match any document in your question. Secondly, you have to devise another means of running the comparison, because the aggregation pipeline in the $addFileds stage will be affected by timezone and other issues as well.
I have a huge mongodb collection and I am trying to export only specific fields from my mongodb but everything is getting exported in csv. I used following query to return all data in moduleName, time and device_ip(field in events array) field
mongoexport --host host-ipaddress --port 27017 --username admin --password password#123 --authenticationDatabase admin --db servername --collection alert --fields 'originalAlert.moduleName:1,originalAlert.time:1,originalAlert.events.0.device_ip:1' --query '{"receivedTime":{$gte:new Date(1583020800000), $lt:new Date(1585612800000)}}' --out /tmp/test.csv
Below is one object from alert collection in mongodb
{
"_id" : ObjectId("5e4bbb89208a6a8a435e064e"),
"receivedTime" : ISODate("2020-02-18T10:25:13.111Z"),
"status" : "GROUPED_IN_INCIDENT",
"originalHeaders" : {
"name" : "Name of the Alert",
"description" : null,
"version" : 0,
"severity" : 5,
"timestamp" : NumberLong(1582021513108),
"signatureId" : "30a9fedd3a7cb83dd66436057dd11445c6adfd242849c3813b38e62399128fd8",
"deviceVendor" : "ABC",
"deviceProduct" : "XYZ",
"deviceVersion" : "123"
},
"originalAlert" : {
"severity" : 5,
"eventSourceId" : "x.x.x.x:50005:406265417822",
"respondEnabled" : true,
"moduleType" : "BASIC",
"engineUri" : "Some Value",
"moduleName" : "Name of the Alert",
"suppressMessageBus" : false,
"transientAlert" : false,
"notificationReasons" : [
"Some-Value",
"Some-Value.2"
],
"actualEventsCount" : 3,
"instanceId" : "30a9fedd3a7cb83dd66436057dd11445c6adfd242849c3813b38e62399128fd8",
"statement" : "Module_5d7ccff0f28050b535cad89b_Alert",
"id" : "9bef15ce-7dc5-4445-838f-79d78d2d6ea6",
"time" : "Feb 18, 2020 10:25:13 AM UTC",
"moduleId" : "5d7ccff0f28050b535cad89b",
"events" : [
{
"msg" : "sshd[4719444]: Failed password for invalid user ISTOPR from x.x.x.x port 58134 ssh2",
"event_byte_size" : 386,
"ec_activity" : "Logon",
"header_id" : "0013",
"alias_host" : [
"some-hostname"
],
"event_cat_name" : "User.Activity.Failed Logins",
"ip_src" : "x.x.x.x",
"device_type" : "aix",
"sessionid" : NumberLong(406265417822),
"medium" : 32,
"inv_context" : [
"audit",
"compliance",
"authentication"
],
"rid" : NumberLong(444833155418),
"feed_name" : [
"investigation"
],
"event_cat" : 1401030000,
"forward_ip" : "x.x.x.x",
"alert_id" : [
"account:logon-failure"
],
"client" : "sshd",
"com_rsa_asoc_streams_source_trail" : [
"admin#x.x.x.x:50005.deployed-rules-sa-managed"
],
"msg_id" : "00003:05",
"device_disc" : 55,
"com_rsa_asoc_streams_stream" : "deployed-rules-sa-managed-stream",
"lc_cid" : "some-id",
"ec_subject" : "User",
"event_source_id" : "x.x.x.x:50005:406265417822",
"com_rsa_asoc_streams_arrival_sequence" : 1789715,
"esa_time" : NumberLong(1582021513102),
"ec_theme" : "Authentication",
"com_rsa_asoc_streams_arrival_timestamp" : NumberLong(1582021512436),
"device_disc_type" : "aix",
"inv_category" : [
"assurance",
"identity"
],
"device_ip" : "x.x.x.x",
"ip_srcport" : 58134,
"event_desc" : "Password failed",
"user_dst" : "invalid user ISTOPR",
"size" : 210,
"netname" : [
"private src"
],
"device_class" : "Unix",
"time" : NumberLong(1582021395000),
"ec_outcome" : "Failure",
"did" : "some-did"
},
{
"msg" : "sshd[4719444]: Failed password for invalid user ISTOPR from x.x.x.x port 58134 ssh2",
"event_byte_size" : 386,
"ec_activity" : "Logon",
"header_id" : "0013",
"alias_host" : [
"some-hostname"
],
"event_cat_name" : "User.Activity.Failed Logins",
"ip_src" : "x.x.x.x",
"device_type" : "aix",
"sessionid" : NumberLong(406265417824),
"medium" : 32,
"inv_context" : [
"audit",
"compliance",
"authentication"
],
"rid" : NumberLong(444833155420),
"feed_name" : [
"investigation"
],
"event_cat" : 1401030000,
"forward_ip" : "x.x.x.x",
"alert_id" : [
"account:logon-failure"
],
"client" : "sshd",
"com_rsa_asoc_streams_source_trail" : [
"admin#x.x.x.x:50005.deployed-rules-sa-managed"
],
"msg_id" : "00003:05",
"device_disc" : 55,
"com_rsa_asoc_streams_stream" : "deployed-rules-sa-managed-stream",
"lc_cid" : "some-id",
"ec_subject" : "User",
"event_source_id" : "x.x.x.x:50005:406265417824",
"com_rsa_asoc_streams_arrival_sequence" : 1789717,
"esa_time" : NumberLong(1582021513103),
"ec_theme" : "Authentication",
"com_rsa_asoc_streams_arrival_timestamp" : NumberLong(1582021512436),
"device_disc_type" : "aix",
"inv_category" : [
"assurance",
"identity"
],
"device_ip" : "x.x.x.x",
"ip_srcport" : 58134,
"event_desc" : "Password failed",
"user_dst" : "invalid user ISTOPR",
"size" : 210,
"netname" : [
"private src"
],
"device_class" : "Unix",
"time" : NumberLong(1582021395000),
"ec_outcome" : "Failure",
"did" : "some-did"
},
{
"msg" : "sshd[4719444]: Failed password for invalid user ISTOPR from x.x.x.x port 58134 ssh2",
"event_byte_size" : 386,
"ec_activity" : "Logon",
"header_id" : "0013",
"alias_host" : [
"some-hostname"
],
"event_cat_name" : "User.Activity.Failed Logins",
"ip_src" : "x.x.x.x",
"device_type" : "aix",
"sessionid" : NumberLong(406265417826),
"medium" : 32,
"inv_context" : [
"audit",
"compliance",
"authentication"
],
"rid" : NumberLong(444833155422),
"feed_name" : [
"investigation"
],
"event_cat" : 1401030000,
"forward_ip" : "x.x.x.x",
"alert_id" : [
"account:logon-failure"
],
"client" : "sshd",
"com_rsa_asoc_streams_source_trail" : [
"admin#x.x.x.x:50005.deployed-rules-sa-managed"
],
"msg_id" : "00003:05",
"device_disc" : 55,
"com_rsa_asoc_streams_stream" : "deployed-rules-sa-managed-stream",
"lc_cid" : "some-id",
"ec_subject" : "User",
"event_source_id" : "x.x.x.x:50005:406265417826",
"com_rsa_asoc_streams_arrival_sequence" : 1789719,
"esa_time" : NumberLong(1582021513103),
"ec_theme" : "Authentication",
"com_rsa_asoc_streams_arrival_timestamp" : NumberLong(1582021512436),
"device_disc_type" : "aix",
"inv_category" : [
"assurance",
"identity"
],
"device_ip" : "x.x.x.x",
"ip_srcport" : 58134,
"event_desc" : "Password failed",
"user_dst" : "invalid user ISTOPR",
"size" : 210,
"netname" : [
"private src"
],
"device_class" : "Unix",
"time" : NumberLong(1582021395000),
"ec_outcome" : "Failure",
"did" : "some-did"
}
],
"suppressNotification" : false
},
"alert" : {
"groupby_source_device_mac_address" : "",
"user_summary" : [
"invalid user ISTOPR"
],
"source" : "Event Stream Analysis",
"type" : [
"Log"
],
"groupby_user_src" : "",
"groupby_source_country" : "",
"grouby_src_device_dns_domain" : "",
"grouby_detector_dns_hostname" : "",
"groupby_analysis_file" : "",
"groupby_filename" : "",
"groupby_source_username" : "",
"groupby_detector_ip" : "x.x.x.x",
"events" : [
{
"agent_id" : "",
"data" : [
{
"filename" : "",
"size" : 210,
"hash" : ""
}
],
"destination" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : "",
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : "invalid user ISTOPR"
},
"hash" : ""
},
"description" : "Password failed",
"domain_src" : "",
"device_type" : "aix",
"event_source" : "x.x.x.x:50005",
"source" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : 58134,
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "x.x.x.x",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : ""
},
"hash" : ""
},
"type" : "Log",
"analysis_file" : "",
"enrichment" : "",
"user_src" : "",
"hostname" : "some-hostname",
"analysis_service" : "",
"file" : "",
"detected_by" : "Unix-aix,x.x.x.x",
"process_vid" : "",
"host_src" : "",
"action" : "",
"operating_system" : "",
"alias_ip" : "",
"from" : "x.x.x.x:58134",
"timestamp" : ISODate("2020-02-18T10:23:15.000Z"),
"event_source_id" : "406265417822",
"related_links" : [
{
"type" : "investigate_original_event",
"url" : "/investigation/host/x.x.x.x:50005/navigate/event/AUTO/406265417822"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"port_dst" : "",
"domain_dst" : "",
"user_dst" : "invalid user ISTOPR",
"host_dst" : "",
"size" : 210,
"domain" : "some-hostname",
"user_account" : "",
"to" : "",
"category" : "",
"detector" : {
"device_class" : "Unix",
"ip_address" : "x.x.x.x",
"product_name" : "aix"
},
"user" : "invalid user ISTOPR",
"analysis_session" : "",
"username" : ""
},
{
"agent_id" : "",
"data" : [
{
"filename" : "",
"size" : 210,
"hash" : ""
}
],
"destination" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : "",
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : "invalid user ISTOPR"
},
"hash" : ""
},
"description" : "Password failed",
"domain_src" : "",
"device_type" : "aix",
"event_source" : "x.x.x.x:50005",
"source" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : 58134,
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "x.x.x.x",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : ""
},
"hash" : ""
},
"type" : "Log",
"analysis_file" : "",
"enrichment" : "",
"user_src" : "",
"hostname" : "some-hostname",
"analysis_service" : "",
"file" : "",
"detected_by" : "Unix-aix,x.x.x.x",
"process_vid" : "",
"host_src" : "",
"action" : "",
"operating_system" : "",
"alias_ip" : "",
"from" : "x.x.x.x:58134",
"timestamp" : ISODate("2020-02-18T10:23:15.000Z"),
"event_source_id" : "406265417824",
"related_links" : [
{
"type" : "investigate_original_event",
"url" : "/investigation/host/x.x.x.x:50005/navigate/event/AUTO/406265417824"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"port_dst" : "",
"domain_dst" : "",
"user_dst" : "invalid user ISTOPR",
"host_dst" : "",
"size" : 210,
"domain" : "some-hostname",
"user_account" : "",
"to" : "",
"category" : "",
"detector" : {
"device_class" : "Unix",
"ip_address" : "x.x.x.x",
"product_name" : "aix"
},
"user" : "invalid user ISTOPR",
"analysis_session" : "",
"username" : ""
},
{
"agent_id" : "",
"data" : [
{
"filename" : "",
"size" : 210,
"hash" : ""
}
],
"destination" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : "",
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : "invalid user ISTOPR"
},
"hash" : ""
},
"description" : "Password failed",
"domain_src" : "",
"device_type" : "aix",
"event_source" : "x.x.x.x:50005",
"source" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : 58134,
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "x.x.x.x",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : ""
},
"hash" : ""
},
"type" : "Log",
"analysis_file" : "",
"enrichment" : "",
"user_src" : "",
"hostname" : "some-hostname",
"analysis_service" : "",
"file" : "",
"detected_by" : "Unix-aix,x.x.x.x",
"process_vid" : "",
"host_src" : "",
"action" : "",
"operating_system" : "",
"alias_ip" : "",
"from" : "x.x.x.x:58134",
"timestamp" : ISODate("2020-02-18T10:23:15.000Z"),
"event_source_id" : "406265417826",
"related_links" : [
{
"type" : "investigate_original_event",
"url" : "/investigation/host/x.x.x.x:50005/navigate/event/AUTO/406265417826"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"port_dst" : "",
"domain_dst" : "",
"user_dst" : "invalid user ISTOPR",
"host_dst" : "",
"size" : 210,
"domain" : "some-hostname",
"user_account" : "",
"to" : "",
"category" : "",
"detector" : {
"device_class" : "Unix",
"ip_address" : "x.x.x.x",
"product_name" : "aix"
},
"user" : "invalid user ISTOPR",
"analysis_session" : "",
"username" : ""
}
],
"grouby_detector_dns_domain" : "",
"host_summary" : [
"x.x.x.x:58134"
],
"groupby_username" : "",
"grouby_src_device_dns_hostname" : "",
"grouby_dst_usr_ad_username" : "",
"groupby_file_sha_256" : "",
"groupby_user_dst" : "invalid user ISTOPR",
"groupby_os" : "",
"grouby_src_usr_ad_domain" : "",
"name" : "Multiple Failed AIX Logins detected",
"groupby_host_src" : "",
"groupby_analysis_service" : "",
"groupby_destination_device_mac_address" : "",
"groupby_version" : "0",
"grouby_src_device_geolocation_domain" : "",
"destination_country" : [],
"groupby_type" : "Log",
"grouby_src_device_netbios_name" : "",
"groupby_device_type" : "aix",
"groupby_domain" : "some-hostname",
"grouby_dst_device_dns_hostname" : "",
"groupby_destination_country" : "",
"grouby_dst_usr_username" : "invalid user ISTOPR",
"grouby_dst_usr_ad_domian" : "",
"groupby_analysis_session" : "",
"signature_id" : "30a9fedd3a7cb83dd66436057dd11445c6adfd242849c3813b38e62399128fd8",
"groupby_data_hash" : "",
"groupby_domain_dst" : "",
"groupby_destination_ip" : "",
"groupby_host_dst" : "",
"grouby_dst_device_geolocation_domain" : "",
"grouby_dst_device_netbios_name" : "",
"groupby_source_ip" : "x.x.x.x",
"groupby_detector_mac_address" : "",
"timestamp" : ISODate("2020-02-18T10:25:13.108Z"),
"severity" : 50.0,
"related_links" : [
{
"type" : "investigate_session",
"url" : "/investigation/x.x.x.x:50005/navigate/query/sessionid%3D406265417822%7C%7Csessionid%3D406265417824%7C%7Csessionid%3D406265417826"
},
{
"type" : "investigate_device_ip",
"url" : "/investigation/x.x.x.x:50005/navigate/query/device.ip%3D10.192.30.44%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
},
{
"type" : "investigate_src_ip",
"url" : "/investigation/x.x.x.x:50005/navigate/query/ip.src%3D10.192.8.167%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"risk_score" : 50.0,
"grouby_dst_device_dns_domain" : "",
"grouby_src_usr_ad_username" : "",
"groupby_destination_port" : "",
"groupby_c2domain" : "",
"groupby_host_name" : "some-hostname",
"source_country" : [],
"groupby_domain_src" : "",
"numEvents" : 3,
"groupby_agent_id" : ""
},
"partOfIncident" : true,
"_class" : "com.rsa.asoc.respond.commons.domain.Alert",
"incidentCreated" : ISODate("2020-02-18T10:25:47.228Z"),
"incidentId" : "INC-1"
}
Sample Output:
Sample Output
Please let me know where I did wrong.
You are missing the --csv argument to mongoexport command.
As of MongoDB v4.2.2, the --csv is deprecated but will work. The more correct option would be to do a --type=csv as an argument to mongoexport
Also, I see there are issues in the way the fields argument and query argument are constructed.
mongoexport --host <host> --port <port> --db <db> --collection <collection> --type=csv --fields 'originalAlert.moduleName,originalAlert.time,originalAlert.events.0.device_ip' --out /tmp/test.csv
Produces results
originalAlert.moduleName,originalAlert.time,originalAlert.events.0.device_ip
Name of the Alert,"Feb 18, 2020 10:25:13 AM UTC",x.x.x.x
On adding the --query, I added a --query '{"receivedTime":{"$gte": {"$date" : "<start-date>"}, "$lt": {"$date":"<end-date>"}}}' to produce :
mongoexport --host <host> --port <port> --db <db> --collection <collection> --type=csv --fields 'originalAlert.moduleName,originalAlert.time,originalAlert.events.0.device_ip' --query '{"receivedTime":{"$gte": {"$date" : "2020-02-18T00:00:01Z"}, "$lt": {"$date":"2020-02-19T00:00:01Z"}}}' --out /tmp/test.csv
This does filter the records appropriately.
My document structure is shown here:
{
"_id" : ObjectId("52517d9e6e0af435ddd48219"),
"date" : ISODate("2013-10-06T18:11:26.329Z"),
"engines" : {
"ahnlab" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "ahnlab",
"node_name" : "",
"status" : 1,
"task_id" : "7fac4f67-2bde-49de-980a-4f7fa1d46db7",
"threat" : "EICAR_Test_File"
},
"avast" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "avast",
"node_name" : "",
"status" : 4,
"task_id" : "fdce50e9-9bf3-4cc9-91a4-b674a108d478",
"threat" : ""
},
"avg" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "avg",
"node_name" : "",
"status" : 1,
"task_id" : "4bdb4a37-80ab-4631-8587-edcbde7e2592",
"threat" : "EICAR_Test"
},
"avira" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "avira",
"node_name" : "",
"status" : 4,
"task_id" : "43b43bab-ff67-440c-9919-f6241ccaf539",
"threat" : ""
},
"bitdefender" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "bitdefender",
"node_name" : "",
"status" : 4,
"task_id" : "ba7efe74-ef93-465f-b6c3-6c4fc746934d",
"threat" : ""
},
"comodo" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "comodo",
"node_name" : "",
"status" : 4,
"task_id" : "831d2dc8-704b-4eb7-9a75-9a0364a8ab09",
"threat" : ""
},
"drweb" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "drweb",
"node_name" : "",
"status" : 4,
"task_id" : "ebb758c3-9146-4a99-b36f-0fb6ee024a33",
"threat" : ""
},
"f-prot" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "f-prot",
"node_name" : "",
"status" : 4,
"task_id" : "13590a25-ac0d-4b1b-b93e-bc715009432a",
"threat" : ""
},
"forticlient" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "forticlient",
"node_name" : "",
"status" : 4,
"task_id" : "",
"threat" : ""
},
"kaspersky" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "kaspersky",
"node_name" : "",
"status" : 4,
"task_id" : "",
"threat" : ""
},
"malwarebytes" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "malwarebytes",
"node_name" : "",
"status" : 4,
"task_id" : "a97f3d9a-d6a9-44df-8355-c053e9f4980a",
"threat" : ""
},
"mcafee" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "mcafee",
"node_name" : "",
"status" : 1,
"task_id" : "e7a592bb-84fc-4c47-a1ed-3719874b19cc",
"threat" : "EICAR test file"
},
"msessentials" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "msessentials",
"node_name" : "",
"status" : 4,
"task_id" : "01c9fb71-155a-473d-b45c-91fa117ae649",
"threat" : ""
},
"nod32" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "nod32",
"node_name" : "",
"status" : 4,
"task_id" : "a715cc8f-0e2f-4698-b883-a35fa6add13e",
"threat" : ""
},
"norman" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "norman",
"node_name" : "",
"status" : 4,
"task_id" : "29da2955-0674-45d2-ac4d-c0b3ea401cba",
"threat" : ""
},
"norton" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "norton",
"node_name" : "",
"status" : 4,
"task_id" : "c4cd1e3e-2f0d-4bf6-84cf-ab0962f7f4ed",
"threat" : ""
},
"panda" : {
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "panda",
"node_name" : "",
"status" : 4,
"task_id" : "b0888f7c-e4a4-4b4e-b163-21d283e166f1",
"threat" : ""
}
},
"expiration_date" : ISODate("2013-10-06T18:11:36.329Z"),
"file_name" : "ffgtr.exe",
"scan_status" : "DONE",
"task_id" : "4ce4ae9e-ef0a-476a-8189-92a5bfe328bd"
}
I would like to project with aggregation the fields 'date' and 'definitions' of every engine.
('definitions' is an embedded field)
I succeeded doing it only if I specify the specific engine:
cursor = collection.aggregate([
{ "$match": { "date": { "$gte": startdate } } },
{'$project': { "def": "$engines.avast.definitions","date":1,"_id": 0 }}
], allowDiskUse=True)
But I would like to run the query for all engines without specifying specific engine, something like:
cursor = collection.aggregate([
{ "$match": { "date": { "$gte": startdate } } },
{'$project': { "def": "$engines.$elemMatch.definitions","date":1,"_id": 0 }}
], allowDiskUse=True)
(which doesn't work)
What I want to find is the average difference between "date" and each "engine" definition date per engine type.
The main problem here is the structure of your data. In order to work with the aggregation framework and also happily work with most other MongoDB query operations and indexing practices then your data should be structured like this instead.
{
"_id" : ObjectId("52517d9e6e0af435ddd48219"),
"date" : ISODate("2013-10-06T18:11:26.329Z"),
"engines" : [
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "ahnlab",
"node_name" : "",
"status" : 1,
"task_id" : "7fac4f67-2bde-49de-980a-4f7fa1d46db7",
"threat" : "EICAR_Test_File"
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "avast",
"node_name" : "",
"status" : 4,
"task_id" : "fdce50e9-9bf3-4cc9-91a4-b674a108d478",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "avg",
"node_name" : "",
"status" : 1,
"task_id" : "4bdb4a37-80ab-4631-8587-edcbde7e2592",
"threat" : "EICAR_Test"
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "avira",
"node_name" : "",
"status" : 4,
"task_id" : "43b43bab-ff67-440c-9919-f6241ccaf539",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "bitdefender",
"node_name" : "",
"status" : 4,
"task_id" : "ba7efe74-ef93-465f-b6c3-6c4fc746934d",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "comodo",
"node_name" : "",
"status" : 4,
"task_id" : "831d2dc8-704b-4eb7-9a75-9a0364a8ab09",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "drweb",
"node_name" : "",
"status" : 4,
"task_id" : "ebb758c3-9146-4a99-b36f-0fb6ee024a33",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "f-prot",
"node_name" : "",
"status" : 4,
"task_id" : "13590a25-ac0d-4b1b-b93e-bc715009432a",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "forticlient",
"node_name" : "",
"status" : 4,
"task_id" : "",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "kaspersky",
"node_name" : "",
"status" : 4,
"task_id" : "",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "malwarebytes",
"node_name" : "",
"status" : 4,
"task_id" : "a97f3d9a-d6a9-44df-8355-c053e9f4980a",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "mcafee",
"node_name" : "",
"status" : 1,
"task_id" : "e7a592bb-84fc-4c47-a1ed-3719874b19cc",
"threat" : "EICAR test file"
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "msessentials",
"node_name" : "",
"status" : 4,
"task_id" : "01c9fb71-155a-473d-b45c-91fa117ae649",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "nod32",
"node_name" : "",
"status" : 4,
"task_id" : "a715cc8f-0e2f-4698-b883-a35fa6add13e",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "norman",
"node_name" : "",
"status" : 4,
"task_id" : "29da2955-0674-45d2-ac4d-c0b3ea401cba",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "norton",
"node_name" : "",
"status" : 4,
"task_id" : "c4cd1e3e-2f0d-4bf6-84cf-ab0962f7f4ed",
"threat" : ""
},
{
"definitions" : ISODate("2000-01-01T00:00:00.000Z"),
"name" : "panda",
"node_name" : "",
"status" : 4,
"task_id" : "b0888f7c-e4a4-4b4e-b163-21d283e166f1",
"threat" : ""
}
],
"expiration_date" : ISODate("2013-10-06T18:11:36.329Z"),
"file_name" : "ffgtr.exe",
"scan_status" : "DONE",
"task_id" : "4ce4ae9e-ef0a-476a-8189-92a5bfe328bd"
}
Then working with that structure is as simple as processing $unwind and then a $group for your results:
db.collection.aggregate([
{ "$unwind": "$engines" },
{ "$group": {
"_id": "$engines.name",
"avgInterval": { "$avg": {
"$subtract": [ "$date", "$engines.definitions" ]
}}
}}
])
The difference will be in milliseconds between two date objects.
Otherwise you are stuck with traversing the object keys in code using JavaScript with mapReduce:
db.collection.mapReduce(
function() {
var doc = this;
Object.keys(doc.engines).forEach(function(key) {
emit( key,
( doc.date.valueOf() -
doc.engines[key].definitions.valueOf())
);
})
},
function(key,values) {
return ( Array.sum(values) / values.length );
},
{ "out": { "inline": 1 } }
);
Both will give you the average value for each engine "name" accross the collection or other query input.
Try to change the structure as it is better suited to your ongoing needs and provides much faster processing.
I am new to mongoDB, and I am stuck at a simple update operation:
Here is the document that I had stored:
{
"_id" : "xyz",
"basicinfo_showcase_components" : {
"display_name" : "ritesh",
"display_email" : "xyz#gmail.comxsa",
"international_phone_number" : "+91 .....",
},
"address_components" : {
"country" : "India",
"postal_code" : "500004",
"state" : "Telangana",
"city" : "Hyderabad",
"subLocality1" : "aaaa",
"subLocality2" : "",
"subLocality3" : "",
"route" : "",
"geometry" : {
"lat" : 17.43143576387407,
"lng" : 78.464432457672046
},
"formatted_address" : "addrr"
}
}
Now I want to updat the address_components in above document:
I tried:
db.portfolio.update({ "_id" : "xyz", "$isolated" : "true" },
{
"address_components" : {
"country" : "Nepal",
"postal_code" : "878799",
"state" : "Telangana",
"city" : "Kathmandu",
"subLocality1" : "xyz",
"subLocality2" : "",
"subLocality3" : "",
"route" : "",
"geometry" : {
"lat" : 17.43143576387407,
"lng" : 78.464432457672046
},
"formatted_address" : "kath..."
}
});
But after updating, the "basicinfo_showcase_components" is eliminated.
ie. the updated doc is:
{
"_id" : "ohris",
"address_components" : {
.......
}
}
Can you please point out the mistake?
In your update query you passed whole document as your update document, if you want to update just one (or more) fields of document you should use $set operator:
db.test.update({ "_id" : "xyz", "$isolated" : "true" },
{
$set : {
"address_components" : {
"country" : "Nepal",
"postal_code" : "878799",
"state" : "Telangana",
"city" : "Kathmandu",
"subLocality1" : "xyz",
"subLocality2" : "",
"subLocality3" : "",
"route" : "",
"geometry" : {
"lat" : 17.43143576387407,
"lng" : 78.464432457672046
},
"formatted_address" : "kath..."
}
}});
We have got an nested object in the mongo db collection e.g. profile object has got array of address and contacts data.
{
"_id" : "f714af57-b4bd-4c1d-b09c-88f71a446262",
"_class" : "com.vaap.dataobject.profilemanagement.CompanyTypeProfile",
"companyCode" : "IBM",
"companyShortName" : "IBM",
"companyTitle" : "IBM",
"companyName" : "IBM",
"companyDescription" : "IBM",
"parentCompanyId" : "",
"companyOwnerName" : "",
"companyPhoneNo" : "",
"companyEmail" : "",
"companyFaxNo" : "",
"companyWebsite" : "",
"companyTotalEmployees" : "",
"companyAnnualRevenue" : "",
"totalPayPeriods" : "",
"payPeriodStartDate" : "",
"companyCreditLimit" : "",
"companyCurrencyId" : "",
"addressData" : [
{
"_id" : "b418982e-4b7a-43c0-aeb6-1ed43ec12f9a",
"addressTypeId" : "efa5d767-41cf-4fa6-b252-70f4113370ea",
"addressTypeName" : "Business",
"addressName" : "IBM, Tysons",
"addressLine1" : "123 Main Street",
"addressLine2" : "",
"countryName" : "United States of America",
"countryId" : "b7520372-6dd7-4d76-94a8-4a5eeee26408",
"stateName" : "Virginia",
"stateId" : "2b155b92-c2f5-44a3-90aa-c188eed38c13",
"cityName" : "Fairfax",
"cityId" : "ed8ae0f4-eff6-46fc-a214-89b3451407bd",
"countyName" : "",
"countyId" : "",
"zipCode" : "20152",
"zipPlusCode" : "",
"ruralCode" : "",
"duration" : "10"
}
],
"contactsData" : [
{
"_id" : "c9389dbd-1b65-4c1a-a9a5-a3c33d89e3b6",
"contactTypeName" : "Business",
"contactTypeId" : "423571d5-5a3a-4f56-9aed-dbb1cef3d9ad",
"prefix" : "Mr.",
"firstName" : "Ravi",
"middleName" : "V",
"lastName" : "Pulipaka",
"nickName" : "Ravi",
"title" : "CEO",
"priorName" : "Ravi",
"primaryContactMethodId" : "ae69fea8-1d54-4b98-813b-0313b7571802",
"primaryContactMethodName" : "Phone",
"primaryPhone" : "703-777-7777",
"primaryEmail" : "",
"primaryFaxNo" : "",
"secondaryFaxNo" : "",
"secondaryPhone" : "",
"businessPhone" : "",
"homePhone" : "",
"personalPhone" : "",
"spousePhone" : "",
"vacationPhone" : "",
"secondaryEmail" : "",
"businessEmail" : "",
"homeEmail" : "",
"personalEmail" : "",
"spouseEmail" : "",
"alternateEmail" : "",
"vacationEmail" : "",
"addressLine1" : "",
"addressLine2" : "",
"countryName" : "",
"countryId" : "",
"stateName" : "",
"stateId" : "",
"cityName" : "",
"cityId" : "",
"countyName" : "",
"countyId" : "",
"zipCode" : "",
"zipPlusCode" : "",
"ruralCode" : ""
}
],
"projectsData" : [],
"recCreBy" : "Rakesh Kumar Goyal",
"recCreDate" : "03-Mar-2014",
"aspectSource" : "AE",
"aspectType" : "Customer",
"clientId" : "VAAP-CLIENT-1"
}
We are able to search profile collection using mongo db full text search. But return results doesn't have a any information as to return records matched exactly with which field in the collection. Is there any way by which I could know the matching field or matching nested object (main profile or contacts data or address data).
I am searching "Ravi" and interested to know whether "Ravi" matched in addressData or contactsData or profile data itself.