Matching object name in nested collection full text search mongodb - mongodb

We have got an nested object in the mongo db collection e.g. profile object has got array of address and contacts data.
{
"_id" : "f714af57-b4bd-4c1d-b09c-88f71a446262",
"_class" : "com.vaap.dataobject.profilemanagement.CompanyTypeProfile",
"companyCode" : "IBM",
"companyShortName" : "IBM",
"companyTitle" : "IBM",
"companyName" : "IBM",
"companyDescription" : "IBM",
"parentCompanyId" : "",
"companyOwnerName" : "",
"companyPhoneNo" : "",
"companyEmail" : "",
"companyFaxNo" : "",
"companyWebsite" : "",
"companyTotalEmployees" : "",
"companyAnnualRevenue" : "",
"totalPayPeriods" : "",
"payPeriodStartDate" : "",
"companyCreditLimit" : "",
"companyCurrencyId" : "",
"addressData" : [
{
"_id" : "b418982e-4b7a-43c0-aeb6-1ed43ec12f9a",
"addressTypeId" : "efa5d767-41cf-4fa6-b252-70f4113370ea",
"addressTypeName" : "Business",
"addressName" : "IBM, Tysons",
"addressLine1" : "123 Main Street",
"addressLine2" : "",
"countryName" : "United States of America",
"countryId" : "b7520372-6dd7-4d76-94a8-4a5eeee26408",
"stateName" : "Virginia",
"stateId" : "2b155b92-c2f5-44a3-90aa-c188eed38c13",
"cityName" : "Fairfax",
"cityId" : "ed8ae0f4-eff6-46fc-a214-89b3451407bd",
"countyName" : "",
"countyId" : "",
"zipCode" : "20152",
"zipPlusCode" : "",
"ruralCode" : "",
"duration" : "10"
}
],
"contactsData" : [
{
"_id" : "c9389dbd-1b65-4c1a-a9a5-a3c33d89e3b6",
"contactTypeName" : "Business",
"contactTypeId" : "423571d5-5a3a-4f56-9aed-dbb1cef3d9ad",
"prefix" : "Mr.",
"firstName" : "Ravi",
"middleName" : "V",
"lastName" : "Pulipaka",
"nickName" : "Ravi",
"title" : "CEO",
"priorName" : "Ravi",
"primaryContactMethodId" : "ae69fea8-1d54-4b98-813b-0313b7571802",
"primaryContactMethodName" : "Phone",
"primaryPhone" : "703-777-7777",
"primaryEmail" : "",
"primaryFaxNo" : "",
"secondaryFaxNo" : "",
"secondaryPhone" : "",
"businessPhone" : "",
"homePhone" : "",
"personalPhone" : "",
"spousePhone" : "",
"vacationPhone" : "",
"secondaryEmail" : "",
"businessEmail" : "",
"homeEmail" : "",
"personalEmail" : "",
"spouseEmail" : "",
"alternateEmail" : "",
"vacationEmail" : "",
"addressLine1" : "",
"addressLine2" : "",
"countryName" : "",
"countryId" : "",
"stateName" : "",
"stateId" : "",
"cityName" : "",
"cityId" : "",
"countyName" : "",
"countyId" : "",
"zipCode" : "",
"zipPlusCode" : "",
"ruralCode" : ""
}
],
"projectsData" : [],
"recCreBy" : "Rakesh Kumar Goyal",
"recCreDate" : "03-Mar-2014",
"aspectSource" : "AE",
"aspectType" : "Customer",
"clientId" : "VAAP-CLIENT-1"
}
We are able to search profile collection using mongo db full text search. But return results doesn't have a any information as to return records matched exactly with which field in the collection. Is there any way by which I could know the matching field or matching nested object (main profile or contacts data or address data).
I am searching "Ravi" and interested to know whether "Ravi" matched in addressData or contactsData or profile data itself.

Related

How to extract specific fields from mongodb in a specific time range?

I have a huge mongodb collection and I am trying to export only specific fields from my mongodb but everything is getting exported in csv. I used following query to return all data in moduleName, time and device_ip(field in events array) field
mongoexport --host host-ipaddress --port 27017 --username admin --password password#123 --authenticationDatabase admin --db servername --collection alert --fields 'originalAlert.moduleName:1,originalAlert.time:1,originalAlert.events.0.device_ip:1' --query '{"receivedTime":{$gte:new Date(1583020800000), $lt:new Date(1585612800000)}}' --out /tmp/test.csv
Below is one object from alert collection in mongodb
{
"_id" : ObjectId("5e4bbb89208a6a8a435e064e"),
"receivedTime" : ISODate("2020-02-18T10:25:13.111Z"),
"status" : "GROUPED_IN_INCIDENT",
"originalHeaders" : {
"name" : "Name of the Alert",
"description" : null,
"version" : 0,
"severity" : 5,
"timestamp" : NumberLong(1582021513108),
"signatureId" : "30a9fedd3a7cb83dd66436057dd11445c6adfd242849c3813b38e62399128fd8",
"deviceVendor" : "ABC",
"deviceProduct" : "XYZ",
"deviceVersion" : "123"
},
"originalAlert" : {
"severity" : 5,
"eventSourceId" : "x.x.x.x:50005:406265417822",
"respondEnabled" : true,
"moduleType" : "BASIC",
"engineUri" : "Some Value",
"moduleName" : "Name of the Alert",
"suppressMessageBus" : false,
"transientAlert" : false,
"notificationReasons" : [
"Some-Value",
"Some-Value.2"
],
"actualEventsCount" : 3,
"instanceId" : "30a9fedd3a7cb83dd66436057dd11445c6adfd242849c3813b38e62399128fd8",
"statement" : "Module_5d7ccff0f28050b535cad89b_Alert",
"id" : "9bef15ce-7dc5-4445-838f-79d78d2d6ea6",
"time" : "Feb 18, 2020 10:25:13 AM UTC",
"moduleId" : "5d7ccff0f28050b535cad89b",
"events" : [
{
"msg" : "sshd[4719444]: Failed password for invalid user ISTOPR from x.x.x.x port 58134 ssh2",
"event_byte_size" : 386,
"ec_activity" : "Logon",
"header_id" : "0013",
"alias_host" : [
"some-hostname"
],
"event_cat_name" : "User.Activity.Failed Logins",
"ip_src" : "x.x.x.x",
"device_type" : "aix",
"sessionid" : NumberLong(406265417822),
"medium" : 32,
"inv_context" : [
"audit",
"compliance",
"authentication"
],
"rid" : NumberLong(444833155418),
"feed_name" : [
"investigation"
],
"event_cat" : 1401030000,
"forward_ip" : "x.x.x.x",
"alert_id" : [
"account:logon-failure"
],
"client" : "sshd",
"com_rsa_asoc_streams_source_trail" : [
"admin#x.x.x.x:50005.deployed-rules-sa-managed"
],
"msg_id" : "00003:05",
"device_disc" : 55,
"com_rsa_asoc_streams_stream" : "deployed-rules-sa-managed-stream",
"lc_cid" : "some-id",
"ec_subject" : "User",
"event_source_id" : "x.x.x.x:50005:406265417822",
"com_rsa_asoc_streams_arrival_sequence" : 1789715,
"esa_time" : NumberLong(1582021513102),
"ec_theme" : "Authentication",
"com_rsa_asoc_streams_arrival_timestamp" : NumberLong(1582021512436),
"device_disc_type" : "aix",
"inv_category" : [
"assurance",
"identity"
],
"device_ip" : "x.x.x.x",
"ip_srcport" : 58134,
"event_desc" : "Password failed",
"user_dst" : "invalid user ISTOPR",
"size" : 210,
"netname" : [
"private src"
],
"device_class" : "Unix",
"time" : NumberLong(1582021395000),
"ec_outcome" : "Failure",
"did" : "some-did"
},
{
"msg" : "sshd[4719444]: Failed password for invalid user ISTOPR from x.x.x.x port 58134 ssh2",
"event_byte_size" : 386,
"ec_activity" : "Logon",
"header_id" : "0013",
"alias_host" : [
"some-hostname"
],
"event_cat_name" : "User.Activity.Failed Logins",
"ip_src" : "x.x.x.x",
"device_type" : "aix",
"sessionid" : NumberLong(406265417824),
"medium" : 32,
"inv_context" : [
"audit",
"compliance",
"authentication"
],
"rid" : NumberLong(444833155420),
"feed_name" : [
"investigation"
],
"event_cat" : 1401030000,
"forward_ip" : "x.x.x.x",
"alert_id" : [
"account:logon-failure"
],
"client" : "sshd",
"com_rsa_asoc_streams_source_trail" : [
"admin#x.x.x.x:50005.deployed-rules-sa-managed"
],
"msg_id" : "00003:05",
"device_disc" : 55,
"com_rsa_asoc_streams_stream" : "deployed-rules-sa-managed-stream",
"lc_cid" : "some-id",
"ec_subject" : "User",
"event_source_id" : "x.x.x.x:50005:406265417824",
"com_rsa_asoc_streams_arrival_sequence" : 1789717,
"esa_time" : NumberLong(1582021513103),
"ec_theme" : "Authentication",
"com_rsa_asoc_streams_arrival_timestamp" : NumberLong(1582021512436),
"device_disc_type" : "aix",
"inv_category" : [
"assurance",
"identity"
],
"device_ip" : "x.x.x.x",
"ip_srcport" : 58134,
"event_desc" : "Password failed",
"user_dst" : "invalid user ISTOPR",
"size" : 210,
"netname" : [
"private src"
],
"device_class" : "Unix",
"time" : NumberLong(1582021395000),
"ec_outcome" : "Failure",
"did" : "some-did"
},
{
"msg" : "sshd[4719444]: Failed password for invalid user ISTOPR from x.x.x.x port 58134 ssh2",
"event_byte_size" : 386,
"ec_activity" : "Logon",
"header_id" : "0013",
"alias_host" : [
"some-hostname"
],
"event_cat_name" : "User.Activity.Failed Logins",
"ip_src" : "x.x.x.x",
"device_type" : "aix",
"sessionid" : NumberLong(406265417826),
"medium" : 32,
"inv_context" : [
"audit",
"compliance",
"authentication"
],
"rid" : NumberLong(444833155422),
"feed_name" : [
"investigation"
],
"event_cat" : 1401030000,
"forward_ip" : "x.x.x.x",
"alert_id" : [
"account:logon-failure"
],
"client" : "sshd",
"com_rsa_asoc_streams_source_trail" : [
"admin#x.x.x.x:50005.deployed-rules-sa-managed"
],
"msg_id" : "00003:05",
"device_disc" : 55,
"com_rsa_asoc_streams_stream" : "deployed-rules-sa-managed-stream",
"lc_cid" : "some-id",
"ec_subject" : "User",
"event_source_id" : "x.x.x.x:50005:406265417826",
"com_rsa_asoc_streams_arrival_sequence" : 1789719,
"esa_time" : NumberLong(1582021513103),
"ec_theme" : "Authentication",
"com_rsa_asoc_streams_arrival_timestamp" : NumberLong(1582021512436),
"device_disc_type" : "aix",
"inv_category" : [
"assurance",
"identity"
],
"device_ip" : "x.x.x.x",
"ip_srcport" : 58134,
"event_desc" : "Password failed",
"user_dst" : "invalid user ISTOPR",
"size" : 210,
"netname" : [
"private src"
],
"device_class" : "Unix",
"time" : NumberLong(1582021395000),
"ec_outcome" : "Failure",
"did" : "some-did"
}
],
"suppressNotification" : false
},
"alert" : {
"groupby_source_device_mac_address" : "",
"user_summary" : [
"invalid user ISTOPR"
],
"source" : "Event Stream Analysis",
"type" : [
"Log"
],
"groupby_user_src" : "",
"groupby_source_country" : "",
"grouby_src_device_dns_domain" : "",
"grouby_detector_dns_hostname" : "",
"groupby_analysis_file" : "",
"groupby_filename" : "",
"groupby_source_username" : "",
"groupby_detector_ip" : "x.x.x.x",
"events" : [
{
"agent_id" : "",
"data" : [
{
"filename" : "",
"size" : 210,
"hash" : ""
}
],
"destination" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : "",
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : "invalid user ISTOPR"
},
"hash" : ""
},
"description" : "Password failed",
"domain_src" : "",
"device_type" : "aix",
"event_source" : "x.x.x.x:50005",
"source" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : 58134,
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "x.x.x.x",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : ""
},
"hash" : ""
},
"type" : "Log",
"analysis_file" : "",
"enrichment" : "",
"user_src" : "",
"hostname" : "some-hostname",
"analysis_service" : "",
"file" : "",
"detected_by" : "Unix-aix,x.x.x.x",
"process_vid" : "",
"host_src" : "",
"action" : "",
"operating_system" : "",
"alias_ip" : "",
"from" : "x.x.x.x:58134",
"timestamp" : ISODate("2020-02-18T10:23:15.000Z"),
"event_source_id" : "406265417822",
"related_links" : [
{
"type" : "investigate_original_event",
"url" : "/investigation/host/x.x.x.x:50005/navigate/event/AUTO/406265417822"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"port_dst" : "",
"domain_dst" : "",
"user_dst" : "invalid user ISTOPR",
"host_dst" : "",
"size" : 210,
"domain" : "some-hostname",
"user_account" : "",
"to" : "",
"category" : "",
"detector" : {
"device_class" : "Unix",
"ip_address" : "x.x.x.x",
"product_name" : "aix"
},
"user" : "invalid user ISTOPR",
"analysis_session" : "",
"username" : ""
},
{
"agent_id" : "",
"data" : [
{
"filename" : "",
"size" : 210,
"hash" : ""
}
],
"destination" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : "",
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : "invalid user ISTOPR"
},
"hash" : ""
},
"description" : "Password failed",
"domain_src" : "",
"device_type" : "aix",
"event_source" : "x.x.x.x:50005",
"source" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : 58134,
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "x.x.x.x",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : ""
},
"hash" : ""
},
"type" : "Log",
"analysis_file" : "",
"enrichment" : "",
"user_src" : "",
"hostname" : "some-hostname",
"analysis_service" : "",
"file" : "",
"detected_by" : "Unix-aix,x.x.x.x",
"process_vid" : "",
"host_src" : "",
"action" : "",
"operating_system" : "",
"alias_ip" : "",
"from" : "x.x.x.x:58134",
"timestamp" : ISODate("2020-02-18T10:23:15.000Z"),
"event_source_id" : "406265417824",
"related_links" : [
{
"type" : "investigate_original_event",
"url" : "/investigation/host/x.x.x.x:50005/navigate/event/AUTO/406265417824"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"port_dst" : "",
"domain_dst" : "",
"user_dst" : "invalid user ISTOPR",
"host_dst" : "",
"size" : 210,
"domain" : "some-hostname",
"user_account" : "",
"to" : "",
"category" : "",
"detector" : {
"device_class" : "Unix",
"ip_address" : "x.x.x.x",
"product_name" : "aix"
},
"user" : "invalid user ISTOPR",
"analysis_session" : "",
"username" : ""
},
{
"agent_id" : "",
"data" : [
{
"filename" : "",
"size" : 210,
"hash" : ""
}
],
"destination" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : "",
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : "invalid user ISTOPR"
},
"hash" : ""
},
"description" : "Password failed",
"domain_src" : "",
"device_type" : "aix",
"event_source" : "x.x.x.x:50005",
"source" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : 58134,
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "x.x.x.x",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : ""
},
"hash" : ""
},
"type" : "Log",
"analysis_file" : "",
"enrichment" : "",
"user_src" : "",
"hostname" : "some-hostname",
"analysis_service" : "",
"file" : "",
"detected_by" : "Unix-aix,x.x.x.x",
"process_vid" : "",
"host_src" : "",
"action" : "",
"operating_system" : "",
"alias_ip" : "",
"from" : "x.x.x.x:58134",
"timestamp" : ISODate("2020-02-18T10:23:15.000Z"),
"event_source_id" : "406265417826",
"related_links" : [
{
"type" : "investigate_original_event",
"url" : "/investigation/host/x.x.x.x:50005/navigate/event/AUTO/406265417826"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"port_dst" : "",
"domain_dst" : "",
"user_dst" : "invalid user ISTOPR",
"host_dst" : "",
"size" : 210,
"domain" : "some-hostname",
"user_account" : "",
"to" : "",
"category" : "",
"detector" : {
"device_class" : "Unix",
"ip_address" : "x.x.x.x",
"product_name" : "aix"
},
"user" : "invalid user ISTOPR",
"analysis_session" : "",
"username" : ""
}
],
"grouby_detector_dns_domain" : "",
"host_summary" : [
"x.x.x.x:58134"
],
"groupby_username" : "",
"grouby_src_device_dns_hostname" : "",
"grouby_dst_usr_ad_username" : "",
"groupby_file_sha_256" : "",
"groupby_user_dst" : "invalid user ISTOPR",
"groupby_os" : "",
"grouby_src_usr_ad_domain" : "",
"name" : "Multiple Failed AIX Logins detected",
"groupby_host_src" : "",
"groupby_analysis_service" : "",
"groupby_destination_device_mac_address" : "",
"groupby_version" : "0",
"grouby_src_device_geolocation_domain" : "",
"destination_country" : [],
"groupby_type" : "Log",
"grouby_src_device_netbios_name" : "",
"groupby_device_type" : "aix",
"groupby_domain" : "some-hostname",
"grouby_dst_device_dns_hostname" : "",
"groupby_destination_country" : "",
"grouby_dst_usr_username" : "invalid user ISTOPR",
"grouby_dst_usr_ad_domian" : "",
"groupby_analysis_session" : "",
"signature_id" : "30a9fedd3a7cb83dd66436057dd11445c6adfd242849c3813b38e62399128fd8",
"groupby_data_hash" : "",
"groupby_domain_dst" : "",
"groupby_destination_ip" : "",
"groupby_host_dst" : "",
"grouby_dst_device_geolocation_domain" : "",
"grouby_dst_device_netbios_name" : "",
"groupby_source_ip" : "x.x.x.x",
"groupby_detector_mac_address" : "",
"timestamp" : ISODate("2020-02-18T10:25:13.108Z"),
"severity" : 50.0,
"related_links" : [
{
"type" : "investigate_session",
"url" : "/investigation/x.x.x.x:50005/navigate/query/sessionid%3D406265417822%7C%7Csessionid%3D406265417824%7C%7Csessionid%3D406265417826"
},
{
"type" : "investigate_device_ip",
"url" : "/investigation/x.x.x.x:50005/navigate/query/device.ip%3D10.192.30.44%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
},
{
"type" : "investigate_src_ip",
"url" : "/investigation/x.x.x.x:50005/navigate/query/ip.src%3D10.192.8.167%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"risk_score" : 50.0,
"grouby_dst_device_dns_domain" : "",
"grouby_src_usr_ad_username" : "",
"groupby_destination_port" : "",
"groupby_c2domain" : "",
"groupby_host_name" : "some-hostname",
"source_country" : [],
"groupby_domain_src" : "",
"numEvents" : 3,
"groupby_agent_id" : ""
},
"partOfIncident" : true,
"_class" : "com.rsa.asoc.respond.commons.domain.Alert",
"incidentCreated" : ISODate("2020-02-18T10:25:47.228Z"),
"incidentId" : "INC-1"
}
Sample Output:
Sample Output
Please let me know where I did wrong.
You are missing the --csv argument to mongoexport command.
As of MongoDB v4.2.2, the --csv is deprecated but will work. The more correct option would be to do a --type=csv as an argument to mongoexport
Also, I see there are issues in the way the fields argument and query argument are constructed.
mongoexport --host <host> --port <port> --db <db> --collection <collection> --type=csv --fields 'originalAlert.moduleName,originalAlert.time,originalAlert.events.0.device_ip' --out /tmp/test.csv
Produces results
originalAlert.moduleName,originalAlert.time,originalAlert.events.0.device_ip
Name of the Alert,"Feb 18, 2020 10:25:13 AM UTC",x.x.x.x
On adding the --query, I added a --query '{"receivedTime":{"$gte": {"$date" : "<start-date>"}, "$lt": {"$date":"<end-date>"}}}' to produce :
mongoexport --host <host> --port <port> --db <db> --collection <collection> --type=csv --fields 'originalAlert.moduleName,originalAlert.time,originalAlert.events.0.device_ip' --query '{"receivedTime":{"$gte": {"$date" : "2020-02-18T00:00:01Z"}, "$lt": {"$date":"2020-02-19T00:00:01Z"}}}' --out /tmp/test.csv
This does filter the records appropriately.

how to copy subdocument into the same document

my need is copy one subdocument from a document and insert into the
same document.But i use forEach and findAndModify it can not insert
into the same document.
document example:
{
"_id" : ObjectId("59b5e84d71ab5580d643d070"),
"modifiedOn" : ISODate("2019-04-03T14:57:22.177+0000"),
"modifiedBy" : "XXX",
"createdOn" : ISODate("2017-09-09T16:33:34.464+0000"),
"createdBy" : "liuyu",
"channelSales" : [
{
"platform" : "amazon",
"channel" : "amazon_ca",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dc1")
},
{
"platform" : "amazon",
"channel" : "amazon_uk",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dc0")
},
{
"platform" : "amazon",
"channel" : "amazon_us",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dbf")
},
{
"platform" : "amazon",
"channel" : "amazon_jp",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dbe")
},
{
"platform" : "amazon",
"channel" : "amazon_de",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dbd")
},
{
"platform" : "amazon",
"channel" : "amazon_es",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dbc")
},
{
"platform" : "amazon",
"channel" : "amazon_fr",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dbb")
},
{
"platform" : "amazon",
"channel" : "amazon_it",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39dba")
},
{
"platform" : "ebay",
"channel" : "ebay_au",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db9")
},
{
"platform" : "ebay",
"channel" : "ebay_de",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db8")
},
{
"platform" : "ebay",
"channel" : "ebay_es",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db7")
},
{
"platform" : "ebay",
"channel" : "ebay_fr",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db6")
},
{
"platform" : "ebay",
"channel" : "ebay_it",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db5")
},
{
"platform" : "ebay",
"channel" : "ebay_uk",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db4")
},
{
"platform" : "ebay",
"channel" : "ebay_us",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("59b5f86aaa0ee15555a39db3")
},
{
"platform" : "walmart",
"channel" : "walmart_us",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5a4d9de2bb1aee844f03e1a6")
},
{
"platform" : "walmart",
"channel" : "walmart_ca",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5a4d9de2bb1aee844f03e1a5")
},
{
"platform" : "amazon",
"channel" : "amazon_au",
"saleStatus" : "T",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5abe095bb1d48d194f6187c0")
},
{
"platform" : "amazon",
"channel" : "amazon_in",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5c9af2776f3dcf04491818f2")
}
],
"statusLevel" : "",
"statusType" : "",
"status" : "A",
"skuId" : "abc001",
"__v" : NumberInt(3)
}
i want to copy:
{
"platform" : "walmart",
"channel" : "walmart_us",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5a4d9de2bb1aee844f03e1a6")
}
and change the "channel" : "walmart_dsv",the other files is the same like this :
{
"platform" : "walmart",
"channel" : "walmart_dsv",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"_id" : ObjectId("5a4d9de2bb1aee844f03e1a6")
}
and insert into the same document.
i use this command:
db.getCollection("0521").aggregate([
{$unwind: "$channelSales"},
{$project: {platform: "$channelSales.platform",
channel: "$channelSales.channel",
saleStatus: "$channelSales.saleStatus",
type: "$channelSales.type",
url: "$channelSales.url",
remark: "$channelSales.remark",
isCaught: "$channelSales.isCaught",
_id: "$channelSales._id"
}},
{ $match : { "channel" : "amazon_us"} }
]).forEach(function(award_team){
if(award_team != null)
{
db.getCollection("0521").findAndModify(
{
query: {_id: award_team._id},
update: { $push: {channelSales: [ {platform: award_team.platform, channel: "walmart_dsv", saleStatus: award_team.saleStatus, type: award_team.type, url: award_team.url, remark: award_team.remark, isCaught: award_team.isCaught, _id: award_team._id }] } },
upsert: true,
});
}
});
but it is add a new document,what can i do?
You are making a mistake in projection:
You are using sub-document _id to update the main document and you are using option {upsert: true} so every time it is inserting a new document in case it does not find _id.
Update your project as below:
db.collection.aggregate([
{$unwind: "$channelSales"},
{$project: {
_id: "$_id", // Main Document Id
platform: "$channelSales.platform",
channel: "$channelSales.channel",
saleStatus: "$channelSales.saleStatus",
type: "$channelSales.type",
url: "$channelSales.url",
remark: "$channelSales.remark",
isCaught: "$channelSales.isCaught",
channelSalesId: "$channelSales._id" // Sub-Document Id (Caannel Sales Id)
}},
{ $match : { "channel" : "amazon_us"} }
])
Now you will get a response like below:
{
"_id" : ObjectId("59b5e84d71ab5580d643d070"),
"platform" : "amazon",
"channel" : "amazon_us",
"saleStatus" : "A",
"type" : "",
"url" : "",
"remark" : "",
"isCaught" : "0",
"channelSalesId" : ObjectId("59b5f86aaa0ee15555a39dbf")
}
Now you can use "award_team._id" to update Document:
db.getCollection("0521").findAndModify(
{
query: {_id: award_team._id},
update: { $push: {channelSales: [ {platform: award_team.platform, channel: "walmart_dsv", saleStatus: award_team.saleStatus, type: award_team.type, url: award_team.url, remark: award_team.remark, isCaught: award_team.isCaught, _id: award_team._id }] } },
upsert: true,
});
}
});

Updating complete embedded document in MongoDB

I am new to mongoDB, and I am stuck at a simple update operation:
Here is the document that I had stored:
{
"_id" : "xyz",
"basicinfo_showcase_components" : {
"display_name" : "ritesh",
"display_email" : "xyz#gmail.comxsa",
"international_phone_number" : "+91 .....",
},
"address_components" : {
"country" : "India",
"postal_code" : "500004",
"state" : "Telangana",
"city" : "Hyderabad",
"subLocality1" : "aaaa",
"subLocality2" : "",
"subLocality3" : "",
"route" : "",
"geometry" : {
"lat" : 17.43143576387407,
"lng" : 78.464432457672046
},
"formatted_address" : "addrr"
}
}
Now I want to updat the address_components in above document:
I tried:
db.portfolio.update({ "_id" : "xyz", "$isolated" : "true" },
{
"address_components" : {
"country" : "Nepal",
"postal_code" : "878799",
"state" : "Telangana",
"city" : "Kathmandu",
"subLocality1" : "xyz",
"subLocality2" : "",
"subLocality3" : "",
"route" : "",
"geometry" : {
"lat" : 17.43143576387407,
"lng" : 78.464432457672046
},
"formatted_address" : "kath..."
}
});
But after updating, the "basicinfo_showcase_components" is eliminated.
ie. the updated doc is:
{
"_id" : "ohris",
"address_components" : {
.......
}
}
Can you please point out the mistake?
In your update query you passed whole document as your update document, if you want to update just one (or more) fields of document you should use $set operator:
db.test.update({ "_id" : "xyz", "$isolated" : "true" },
{
$set : {
"address_components" : {
"country" : "Nepal",
"postal_code" : "878799",
"state" : "Telangana",
"city" : "Kathmandu",
"subLocality1" : "xyz",
"subLocality2" : "",
"subLocality3" : "",
"route" : "",
"geometry" : {
"lat" : 17.43143576387407,
"lng" : 78.464432457672046
},
"formatted_address" : "kath..."
}
}});

throw error Duplicate element Name in mongodb c# and How can i achieve nested AND/OR condition?

I have mongo query which i try to retrive some data but it will display error:
"Duplicate Element Name" in MVC4.0.
StringBuilder sb = new StringBuilder();List<BsonDocument> operations = new List<BsonDocument>(); sb.Append("{\"$match\":{\"$and\": [{\"Final_Status\": \"Completed\"},{\"Final_Status\": \"Ongoing\"}],\"$and\": [{\"$or\": [{\"Final_Status\": \"Confirmed\"},{\"Final_Status\": \"Specified\"}]},{\"$or\": [{\"Final_Status\": \"Lost\"}]}]}}");
operations.Add(new BsonDocument(BsonSerializer.Deserialize(sb.ToString())));
var dlist = tempdatatoInsert.Aggregate(operations.ToArray());
after query execution "dlist" contains the result data.
My DB Structure
/* 0 */{ "_id" : ObjectId("5474456cf3e7e912ac43a898"), "ID" : 1, "Discovery_Month_Year" : "May-11", "Complete_Incomplete" : "", "Portal" : "", "Commercial_residental" : "", "Meeting_Not_Yet_Yes_0" : "", "Sample_Flat" : "No", "Project_Name" : "", "Address" : "", "Contact_Number" : "", "Place" : "", "Area" : "", "Stop" : "", "Builder_Group" : "Cosmos Group", "Builder_Category" : "", "Stage_of_Construction" : "Foundation", "No_of_towers" : 1, "Floor" : 30, "Total_units" : 0, "Tentative_completion_month_year" : "Dec-12", "Windows_Type" : "Aluminium Sliding", "Windows_Area" : 3000, "Contact_Person" : "Ketan Sagar", "Field23" : 0, "Contact_Person_Category" : "Purchase", "Decision_Maker" : "", "Project_status_Specification" : "Specified After", "Change_Product_specification" : "", "Specified_Product" : "", "Thickness" : ", "Approx_tonnage" : 0, "Pitching_Product" : "", "Builder_Name" : "", "Builder_contact_no" : "", "Fabricator_Firm" : "", "Fabricator_Name" : "", "Fabricator_Contact" : "", "Architect_Firm" : "", "Architect_Name" : "", "Architect_Contact" : "", "Dealer_Association" : "","Next_Meeting__Follow_Up_Date" : "Feb-12", "Next_Meeting_Follow_up_Agenda" : "", "Remarks_about_Meeting" : "", "Competiton_Presence" : "", "Final_Usage_Company" : "", "Final_Usage_Product" : "", "Scheduled_Plan_Month_Year_1st_Lot" : "13/Apr/2013", Scheduled_qty_in_1st_Lot" : 3000, "Scheduled_Plan_Month_Year_2nd_Lot" : "", Scheduled_qty_in_2nd_Lot" : "", "Scheduled_Dealer" : "", "Window_price_with_glass" : "", "Price_of_Flat_Per_Sq_ft" : 0, "Price_band_Flat" : "", "Remarks" : "", "Final_Status" : Prospecting", "if_Lost_Reason" : "", "Status" : "Cold", "Notes" : "", "Attachment" : ""}
sample Sql Query which i want to convert into MongoDB query
SELECT * FROM [TestDB].[dbo].[EmpTbl] WHERE (Final_Status='Prospecting' and Final_Status='Ongoing') and (Final_Status='Completed' or Final_Status='Specified') or (Final_Status='Confirmed').
You can not have multiple $and at the same level in the json document
try to rewrite your query as
{"$match":{"$and": [{"Final_Status": "Completed"},{"Final_Status": "Ongoing"},{
"$or":[
{
"Final_Status":"Confirmed"
},
{
"Final_Status":"Specified"
}
]
},
{
"$or":[
{
"Final_Status":"Lost"
}
]
}]}}

index search returns all subdocuments

I would like to only return my embedded documents titled 'Listings', based on the text search query. Where may I be going wrong? Creation of the index?
Here is my index:
db.Collection.ensureIndex({"Listings.Title": "text", "Listings.Description" : "text"}, {name: "Search"})
This returns the whole object, only want listings
db.AspNetUsers.runCommand("text", { search: "lawn" })
this returns just listings, but all the listings are included. Not the listings based on the search criteria e.g. 'lawn'
db.AspNetUsers.runCommand("text", { search: "lawn", project: {Listings:1}})
here is my object
{
"_id" : ,
"UserName" : "",
"PasswordHash" : "",
"SecurityStamp" : "",
"Roles" : [],
"Claims" : [],
"Logins" : [],
"ProfileData" : {
"BirthDate" : new Date("3/8/1974 00:00:00"),
"FirstName" : "",
"LastName" : "",
"MiddleName" : "",
"Address" : "",
"Address1" : ,
"City" : "",
"State" : "",
"PostalCode" : "",
"CellPhone" : "",
"HomePhone" : "",
"Location" : {
"type" : "Point",
"coordinates" : [, ]
}
},
"Email" : "",
"ConfirmationToken" : "Confirmed",
"IsConfirmed" : true,
"Listings" : [{
"_id" : ObjectId("5331ac28a5eabf2854085df5"),
"UserId" : ObjectId("5329b43fa5eabf0548490c27"),
"Title" : "Lawn Chairs",
"Description" : "lawn chairs",
"Pictures" : ["5331ac28a5eabf2854085df6", "5331ac28a5eabf2854085df7", "5331ac28a5eabf2854085df8"],
"Category" : {
"_id" : ObjectId("53273ce37dd6c71e1859ab77"),
"Title" : "Leisure"
}
}, {
"_id" : ObjectId("5331ac50a5eabf2854085df9"),
"UserId" : ObjectId("5329b43fa5eabf0548490c27"),
"Title" : "Lawn Ornaments",
"Description" : "lawn ornaments troll frog gnome",
"Pictures" : ["5331ac50a5eabf2854085dfa", "5331ac50a5eabf2854085dfb", "5331ac51a5eabf2854085dfc"],
"Category" : {
"_id" : ObjectId("53273cd57dd6c71e1859ab76"),
"Title" : "Home"
}
}, {
"_id" : ObjectId("5331ac71a5eabf2854085dfd"),
"UserId" : ObjectId("5329b43fa5eabf0548490c27"),
"Title" : "Cell Phone",
"Description" : "Samsung Galaxy S4",
"Pictures" : ["5331ac71a5eabf2854085dfe", "5331ac71a5eabf2854085dff", "5331ac72a5eabf2854085e00"],
"Category" : {
"_id" : ObjectId("53273cd57dd6c71e1859ab76"),
"Title" : "Home"
}
}]
}
You can get just the fields you want using a project parameter. For getting just the Listings element, you could try:
db.AspNetUsers.runCommand("text", {search:"lawn", project:{_id:0, Listings:1}})
EDIT:
The text command will return all documents that contain the search term. You'll get the entire document. If you further want to filter an array inside of the document, you can use the $elemMatch projection operator, in combination with $regex. For example:
db.AspNetUsers.runCommand("text", {
search: "lawn",
project: {_id:0, Listings:{$elemMatch:{ "Title": /lawn/i }}}
})