How to extract specific fields from mongodb in a specific time range? - mongodb

Question

I have a huge mongodb collection and I am trying to export only specific fields from my mongodb but everything is getting exported in csv. I used following query to return all data in moduleName, time and device_ip(field in events array) field
mongoexport --host host-ipaddress --port 27017 --username admin --password password#123 --authenticationDatabase admin --db servername --collection alert --fields 'originalAlert.moduleName:1,originalAlert.time:1,originalAlert.events.0.device_ip:1' --query '{"receivedTime":{$gte:new Date(1583020800000), $lt:new Date(1585612800000)}}' --out /tmp/test.csv
Below is one object from alert collection in mongodb
{
"_id" : ObjectId("5e4bbb89208a6a8a435e064e"),
"receivedTime" : ISODate("2020-02-18T10:25:13.111Z"),
"status" : "GROUPED_IN_INCIDENT",
"originalHeaders" : {
"name" : "Name of the Alert",
"description" : null,
"version" : 0,
"severity" : 5,
"timestamp" : NumberLong(1582021513108),
"signatureId" : "30a9fedd3a7cb83dd66436057dd11445c6adfd242849c3813b38e62399128fd8",
"deviceVendor" : "ABC",
"deviceProduct" : "XYZ",
"deviceVersion" : "123"
},
"originalAlert" : {
"severity" : 5,
"eventSourceId" : "x.x.x.x:50005:406265417822",
"respondEnabled" : true,
"moduleType" : "BASIC",
"engineUri" : "Some Value",
"moduleName" : "Name of the Alert",
"suppressMessageBus" : false,
"transientAlert" : false,
"notificationReasons" : [
"Some-Value",
"Some-Value.2"
],
"actualEventsCount" : 3,
"instanceId" : "30a9fedd3a7cb83dd66436057dd11445c6adfd242849c3813b38e62399128fd8",
"statement" : "Module_5d7ccff0f28050b535cad89b_Alert",
"id" : "9bef15ce-7dc5-4445-838f-79d78d2d6ea6",
"time" : "Feb 18, 2020 10:25:13 AM UTC",
"moduleId" : "5d7ccff0f28050b535cad89b",
"events" : [
{
"msg" : "sshd[4719444]: Failed password for invalid user ISTOPR from x.x.x.x port 58134 ssh2",
"event_byte_size" : 386,
"ec_activity" : "Logon",
"header_id" : "0013",
"alias_host" : [
"some-hostname"
],
"event_cat_name" : "User.Activity.Failed Logins",
"ip_src" : "x.x.x.x",
"device_type" : "aix",
"sessionid" : NumberLong(406265417822),
"medium" : 32,
"inv_context" : [
"audit",
"compliance",
"authentication"
],
"rid" : NumberLong(444833155418),
"feed_name" : [
"investigation"
],
"event_cat" : 1401030000,
"forward_ip" : "x.x.x.x",
"alert_id" : [
"account:logon-failure"
],
"client" : "sshd",
"com_rsa_asoc_streams_source_trail" : [
"admin#x.x.x.x:50005.deployed-rules-sa-managed"
],
"msg_id" : "00003:05",
"device_disc" : 55,
"com_rsa_asoc_streams_stream" : "deployed-rules-sa-managed-stream",
"lc_cid" : "some-id",
"ec_subject" : "User",
"event_source_id" : "x.x.x.x:50005:406265417822",
"com_rsa_asoc_streams_arrival_sequence" : 1789715,
"esa_time" : NumberLong(1582021513102),
"ec_theme" : "Authentication",
"com_rsa_asoc_streams_arrival_timestamp" : NumberLong(1582021512436),
"device_disc_type" : "aix",
"inv_category" : [
"assurance",
"identity"
],
"device_ip" : "x.x.x.x",
"ip_srcport" : 58134,
"event_desc" : "Password failed",
"user_dst" : "invalid user ISTOPR",
"size" : 210,
"netname" : [
"private src"
],
"device_class" : "Unix",
"time" : NumberLong(1582021395000),
"ec_outcome" : "Failure",
"did" : "some-did"
},
{
"msg" : "sshd[4719444]: Failed password for invalid user ISTOPR from x.x.x.x port 58134 ssh2",
"event_byte_size" : 386,
"ec_activity" : "Logon",
"header_id" : "0013",
"alias_host" : [
"some-hostname"
],
"event_cat_name" : "User.Activity.Failed Logins",
"ip_src" : "x.x.x.x",
"device_type" : "aix",
"sessionid" : NumberLong(406265417824),
"medium" : 32,
"inv_context" : [
"audit",
"compliance",
"authentication"
],
"rid" : NumberLong(444833155420),
"feed_name" : [
"investigation"
],
"event_cat" : 1401030000,
"forward_ip" : "x.x.x.x",
"alert_id" : [
"account:logon-failure"
],
"client" : "sshd",
"com_rsa_asoc_streams_source_trail" : [
"admin#x.x.x.x:50005.deployed-rules-sa-managed"
],
"msg_id" : "00003:05",
"device_disc" : 55,
"com_rsa_asoc_streams_stream" : "deployed-rules-sa-managed-stream",
"lc_cid" : "some-id",
"ec_subject" : "User",
"event_source_id" : "x.x.x.x:50005:406265417824",
"com_rsa_asoc_streams_arrival_sequence" : 1789717,
"esa_time" : NumberLong(1582021513103),
"ec_theme" : "Authentication",
"com_rsa_asoc_streams_arrival_timestamp" : NumberLong(1582021512436),
"device_disc_type" : "aix",
"inv_category" : [
"assurance",
"identity"
],
"device_ip" : "x.x.x.x",
"ip_srcport" : 58134,
"event_desc" : "Password failed",
"user_dst" : "invalid user ISTOPR",
"size" : 210,
"netname" : [
"private src"
],
"device_class" : "Unix",
"time" : NumberLong(1582021395000),
"ec_outcome" : "Failure",
"did" : "some-did"
},
{
"msg" : "sshd[4719444]: Failed password for invalid user ISTOPR from x.x.x.x port 58134 ssh2",
"event_byte_size" : 386,
"ec_activity" : "Logon",
"header_id" : "0013",
"alias_host" : [
"some-hostname"
],
"event_cat_name" : "User.Activity.Failed Logins",
"ip_src" : "x.x.x.x",
"device_type" : "aix",
"sessionid" : NumberLong(406265417826),
"medium" : 32,
"inv_context" : [
"audit",
"compliance",
"authentication"
],
"rid" : NumberLong(444833155422),
"feed_name" : [
"investigation"
],
"event_cat" : 1401030000,
"forward_ip" : "x.x.x.x",
"alert_id" : [
"account:logon-failure"
],
"client" : "sshd",
"com_rsa_asoc_streams_source_trail" : [
"admin#x.x.x.x:50005.deployed-rules-sa-managed"
],
"msg_id" : "00003:05",
"device_disc" : 55,
"com_rsa_asoc_streams_stream" : "deployed-rules-sa-managed-stream",
"lc_cid" : "some-id",
"ec_subject" : "User",
"event_source_id" : "x.x.x.x:50005:406265417826",
"com_rsa_asoc_streams_arrival_sequence" : 1789719,
"esa_time" : NumberLong(1582021513103),
"ec_theme" : "Authentication",
"com_rsa_asoc_streams_arrival_timestamp" : NumberLong(1582021512436),
"device_disc_type" : "aix",
"inv_category" : [
"assurance",
"identity"
],
"device_ip" : "x.x.x.x",
"ip_srcport" : 58134,
"event_desc" : "Password failed",
"user_dst" : "invalid user ISTOPR",
"size" : 210,
"netname" : [
"private src"
],
"device_class" : "Unix",
"time" : NumberLong(1582021395000),
"ec_outcome" : "Failure",
"did" : "some-did"
}
],
"suppressNotification" : false
},
"alert" : {
"groupby_source_device_mac_address" : "",
"user_summary" : [
"invalid user ISTOPR"
],
"source" : "Event Stream Analysis",
"type" : [
"Log"
],
"groupby_user_src" : "",
"groupby_source_country" : "",
"grouby_src_device_dns_domain" : "",
"grouby_detector_dns_hostname" : "",
"groupby_analysis_file" : "",
"groupby_filename" : "",
"groupby_source_username" : "",
"groupby_detector_ip" : "x.x.x.x",
"events" : [
{
"agent_id" : "",
"data" : [
{
"filename" : "",
"size" : 210,
"hash" : ""
}
],
"destination" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : "",
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : "invalid user ISTOPR"
},
"hash" : ""
},
"description" : "Password failed",
"domain_src" : "",
"device_type" : "aix",
"event_source" : "x.x.x.x:50005",
"source" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : 58134,
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "x.x.x.x",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : ""
},
"hash" : ""
},
"type" : "Log",
"analysis_file" : "",
"enrichment" : "",
"user_src" : "",
"hostname" : "some-hostname",
"analysis_service" : "",
"file" : "",
"detected_by" : "Unix-aix,x.x.x.x",
"process_vid" : "",
"host_src" : "",
"action" : "",
"operating_system" : "",
"alias_ip" : "",
"from" : "x.x.x.x:58134",
"timestamp" : ISODate("2020-02-18T10:23:15.000Z"),
"event_source_id" : "406265417822",
"related_links" : [
{
"type" : "investigate_original_event",
"url" : "/investigation/host/x.x.x.x:50005/navigate/event/AUTO/406265417822"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"port_dst" : "",
"domain_dst" : "",
"user_dst" : "invalid user ISTOPR",
"host_dst" : "",
"size" : 210,
"domain" : "some-hostname",
"user_account" : "",
"to" : "",
"category" : "",
"detector" : {
"device_class" : "Unix",
"ip_address" : "x.x.x.x",
"product_name" : "aix"
},
"user" : "invalid user ISTOPR",
"analysis_session" : "",
"username" : ""
},
{
"agent_id" : "",
"data" : [
{
"filename" : "",
"size" : 210,
"hash" : ""
}
],
"destination" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : "",
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : "invalid user ISTOPR"
},
"hash" : ""
},
"description" : "Password failed",
"domain_src" : "",
"device_type" : "aix",
"event_source" : "x.x.x.x:50005",
"source" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : 58134,
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "x.x.x.x",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : ""
},
"hash" : ""
},
"type" : "Log",
"analysis_file" : "",
"enrichment" : "",
"user_src" : "",
"hostname" : "some-hostname",
"analysis_service" : "",
"file" : "",
"detected_by" : "Unix-aix,x.x.x.x",
"process_vid" : "",
"host_src" : "",
"action" : "",
"operating_system" : "",
"alias_ip" : "",
"from" : "x.x.x.x:58134",
"timestamp" : ISODate("2020-02-18T10:23:15.000Z"),
"event_source_id" : "406265417824",
"related_links" : [
{
"type" : "investigate_original_event",
"url" : "/investigation/host/x.x.x.x:50005/navigate/event/AUTO/406265417824"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"port_dst" : "",
"domain_dst" : "",
"user_dst" : "invalid user ISTOPR",
"host_dst" : "",
"size" : 210,
"domain" : "some-hostname",
"user_account" : "",
"to" : "",
"category" : "",
"detector" : {
"device_class" : "Unix",
"ip_address" : "x.x.x.x",
"product_name" : "aix"
},
"user" : "invalid user ISTOPR",
"analysis_session" : "",
"username" : ""
},
{
"agent_id" : "",
"data" : [
{
"filename" : "",
"size" : 210,
"hash" : ""
}
],
"destination" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : "",
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : "invalid user ISTOPR"
},
"hash" : ""
},
"description" : "Password failed",
"domain_src" : "",
"device_type" : "aix",
"event_source" : "x.x.x.x:50005",
"source" : {
"path" : "",
"file_SHA256" : "",
"filename" : "",
"launch_argument" : "",
"device" : {
"compliance_rating" : "",
"netbios_name" : "",
"port" : 58134,
"mac_address" : "",
"criticality" : "",
"asset_type" : "",
"ip_address" : "x.x.x.x",
"facility" : "",
"business_unit" : "",
"geolocation" : {
"country" : "",
"city" : "",
"latitude" : null,
"organization" : "",
"domain" : "",
"longitude" : null
}
},
"user" : {
"email_address" : "",
"ad_username" : "",
"ad_domain" : "",
"username" : ""
},
"hash" : ""
},
"type" : "Log",
"analysis_file" : "",
"enrichment" : "",
"user_src" : "",
"hostname" : "some-hostname",
"analysis_service" : "",
"file" : "",
"detected_by" : "Unix-aix,x.x.x.x",
"process_vid" : "",
"host_src" : "",
"action" : "",
"operating_system" : "",
"alias_ip" : "",
"from" : "x.x.x.x:58134",
"timestamp" : ISODate("2020-02-18T10:23:15.000Z"),
"event_source_id" : "406265417826",
"related_links" : [
{
"type" : "investigate_original_event",
"url" : "/investigation/host/x.x.x.x:50005/navigate/event/AUTO/406265417826"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"port_dst" : "",
"domain_dst" : "",
"user_dst" : "invalid user ISTOPR",
"host_dst" : "",
"size" : 210,
"domain" : "some-hostname",
"user_account" : "",
"to" : "",
"category" : "",
"detector" : {
"device_class" : "Unix",
"ip_address" : "x.x.x.x",
"product_name" : "aix"
},
"user" : "invalid user ISTOPR",
"analysis_session" : "",
"username" : ""
}
],
"grouby_detector_dns_domain" : "",
"host_summary" : [
"x.x.x.x:58134"
],
"groupby_username" : "",
"grouby_src_device_dns_hostname" : "",
"grouby_dst_usr_ad_username" : "",
"groupby_file_sha_256" : "",
"groupby_user_dst" : "invalid user ISTOPR",
"groupby_os" : "",
"grouby_src_usr_ad_domain" : "",
"name" : "Multiple Failed AIX Logins detected",
"groupby_host_src" : "",
"groupby_analysis_service" : "",
"groupby_destination_device_mac_address" : "",
"groupby_version" : "0",
"grouby_src_device_geolocation_domain" : "",
"destination_country" : [],
"groupby_type" : "Log",
"grouby_src_device_netbios_name" : "",
"groupby_device_type" : "aix",
"groupby_domain" : "some-hostname",
"grouby_dst_device_dns_hostname" : "",
"groupby_destination_country" : "",
"grouby_dst_usr_username" : "invalid user ISTOPR",
"grouby_dst_usr_ad_domian" : "",
"groupby_analysis_session" : "",
"signature_id" : "30a9fedd3a7cb83dd66436057dd11445c6adfd242849c3813b38e62399128fd8",
"groupby_data_hash" : "",
"groupby_domain_dst" : "",
"groupby_destination_ip" : "",
"groupby_host_dst" : "",
"grouby_dst_device_geolocation_domain" : "",
"grouby_dst_device_netbios_name" : "",
"groupby_source_ip" : "x.x.x.x",
"groupby_detector_mac_address" : "",
"timestamp" : ISODate("2020-02-18T10:25:13.108Z"),
"severity" : 50.0,
"related_links" : [
{
"type" : "investigate_session",
"url" : "/investigation/x.x.x.x:50005/navigate/query/sessionid%3D406265417822%7C%7Csessionid%3D406265417824%7C%7Csessionid%3D406265417826"
},
{
"type" : "investigate_device_ip",
"url" : "/investigation/x.x.x.x:50005/navigate/query/device.ip%3D10.192.30.44%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
},
{
"type" : "investigate_src_ip",
"url" : "/investigation/x.x.x.x:50005/navigate/query/ip.src%3D10.192.8.167%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
},
{
"type" : "investigate_destination_domain",
"url" : "/investigation/x.x.x.x:50005/navigate/query/alias.host%3D'some-hostname'%2Fdate%2F2020-02-18T10%3A13%3A15.000Z%2F2020-02-18T10%3A33%3A15.000Z"
}
],
"risk_score" : 50.0,
"grouby_dst_device_dns_domain" : "",
"grouby_src_usr_ad_username" : "",
"groupby_destination_port" : "",
"groupby_c2domain" : "",
"groupby_host_name" : "some-hostname",
"source_country" : [],
"groupby_domain_src" : "",
"numEvents" : 3,
"groupby_agent_id" : ""
},
"partOfIncident" : true,
"_class" : "com.rsa.asoc.respond.commons.domain.Alert",
"incidentCreated" : ISODate("2020-02-18T10:25:47.228Z"),
"incidentId" : "INC-1"
}
Sample Output:
Sample Output
Please let me know where I did wrong.

Answer 1

You are missing the --csv argument to mongoexport command.
As of MongoDB v4.2.2, the --csv is deprecated but will work. The more correct option would be to do a --type=csv as an argument to mongoexport
Also, I see there are issues in the way the fields argument and query argument are constructed.
mongoexport --host <host> --port <port> --db <db> --collection <collection> --type=csv --fields 'originalAlert.moduleName,originalAlert.time,originalAlert.events.0.device_ip' --out /tmp/test.csv
Produces results
originalAlert.moduleName,originalAlert.time,originalAlert.events.0.device_ip
Name of the Alert,"Feb 18, 2020 10:25:13 AM UTC",x.x.x.x
On adding the --query, I added a --query '{"receivedTime":{"$gte": {"$date" : "<start-date>"}, "$lt": {"$date":"<end-date>"}}}' to produce :
mongoexport --host <host> --port <port> --db <db> --collection <collection> --type=csv --fields 'originalAlert.moduleName,originalAlert.time,originalAlert.events.0.device_ip' --query '{"receivedTime":{"$gte": {"$date" : "2020-02-18T00:00:01Z"}, "$lt": {"$date":"2020-02-19T00:00:01Z"}}}' --out /tmp/test.csv
This does filter the records appropriately.