How to sum two variable in REST API - rest

I want to sum two variable in REST API,and order by it.
This is my REST API:
"aggs": {
"genres": {
"terms": {
"field": "L7_PROTO_NAME.keyword",
"order": {
"sum_bytes": "desc"
}
},
"aggs": {
"in_bytes": {
"sum": {
"field": "IN_BYTES"
}
},
"out_bytes": {
"sum": {
"field": "OUT_BYTES"
}
}
}
thank you in advance!

You need to create another sub-aggregation that sums the two fields and then order the terms aggregation by that sub-aggregation:
{
"query": {
"bool": {
"should": [
{
"term": {
"_index": "logstash-2018.01.02"
}
},
{
"term": {
"IPV4_DST_ADDR": "192.168.0.159"
}
},
{
"term": {
"IPV4_SRC_ADDR": "192.168.0.159"
}
}
]
}
},
"aggs": {
"genres": {
"terms": {
"field": "L7_PROTO_NAME.keyword",
"order": {
"sum_bytes": "desc"
}
},
"aggs": {
"in_bytes": {
"sum": {
"field": "IN_BYTES"
}
},
"out_bytes": {
"sum": {
"field": "OUT_BYTES"
}
},
"sum_bytes": {
"sum": {
"script": {
"source": "doc.IN_BYTES.value + doc.OUT_BYTES.value"
}
}
}
}
}
}
}
Since scripts are quite computation heavy, you should sum those two fields at indexing time and index the result as a new field that you can use directly in your aggregation, like this:
{
"query": {
"bool": {
"should": [
{
"term": {
"_index": "logstash-2018.01.02"
}
},
{
"term": {
"IPV4_DST_ADDR": "192.168.0.159"
}
},
{
"term": {
"IPV4_SRC_ADDR": "192.168.0.159"
}
}
]
}
},
"aggs": {
"genres": {
"terms": {
"field": "L7_PROTO_NAME.keyword",
"order": {
"sum_bytes": "desc"
}
},
"aggs": {
"in_bytes": {
"sum": {
"field": "IN_BYTES"
}
},
"out_bytes": {
"sum": {
"field": "OUT_BYTES"
}
},
"sum_bytes": {
"sum": {
"field": "SUM_BYTES"
}
}
}
}
}
}

Related

How to add an extra attribute in a sub document in MongoDB

How to update/add sub-document in sub-document in mongodb.
I have sample data like this:
{
"baselineParty": {
"AP": [
{
"party": {
"partyId": {
"value": "12345"
},
}
},
{
"party": {
"partyId": {
"value": "12346"
}
}
},
{
"party": {
"partyId": {
"value": "12347"
}
}
}
]
}
}
I want to add an extra field "baselineParty.AP.party.verifiedStatusYn" to the existing json
Expected result would be like this :
{
"baselineParty": {
"AP": [
{
"party": {
"partyId": {
"value": "12345"
}
},
"verifiedStatusYn": {
"by": "cdd",
"updated": "22",
"value": "yes"
}
},
{
"party": {
"partyId": {
"value": "12346"
}
},
"verifiedStatusYn": {
"by": "cdd",
"updated": "22",
"value": "yes"
}
},
{
"party": {
"partyId": {
"value": "12347"
}
},
"verifiedStatusYn": {
"by": "cdd",
"updated": "22",
"value": "yes"
}
}
]
}
}
I tried using $set but not getting expected result.
Do you guys have a solution for this. Thanks!
You will need $[] to update all entries in the array.
db.collection.update({},
{
$set: {
"baselineParty.AP.$[].party.verifiedStatusYn": {
"by": "cdd",
"updated": "22",
"value": "yes"
}
}
})
Here is the Mongo playground for your reference.

Kibana: filter results by aggregated min and max dates from matched id

I want to pass an event_id to Kibana/Elastic Search and find the min and max dates from the #timestamp field for this event_id. Then I want to set the date range to these dates and show all the results. I assume this is doable.
I can get the min and max with this aggregation:
GET /filebeat-*/_search
{
"query": {
"match": {
"event_id": 1234
}
},
"aggs" : {
"min_date": {"min": {"field": "#timestamp" }},
"max_date": {"max": {"field": "#timestamp" }}
}
}
and I can get the results by searching for the specific date range:
GET /filebeat-*/_search
{
"query": {
"bool": {
"filter": {
"range": {"#timestamp": {"gte": "2020-09-11T13:35:35.000Z", "lte": "2020-09-24T20:35:07.000Z"}}
}
}
}
}
how can I combine the two so that I can just change the event_id and have an auto date range type feature?
EDIT:
I can do this:
GET /filebeat-*/_search
{
"query": {
"bool": {
"must": {
"match": {
"event_id": 1234
}
},
"filter": {
"range": {
"#timestamp": {
"lte": "2020-09-25",
"gte": "2020-09-24"
}
}
}
}
},
"aggs": {
"min_date": {
"min": {
"field": "#timestamp"
}
},
"max_date": {
"max": {
"field": "#timestamp"
}
}
}
}
But what I would like to do is something like:
GET /filebeat-*/_search
{
"query": {
"bool": {
"must": {
"match": {
"event_id": 1234
}
},
"filter": {
"range": {
"#timestamp": {
"lte": "max_date",
"gte": "min_date"
}
}
}
}
},
"aggs": {
"min_date": {
"min": {
"field": "#timestamp"
}
},
"max_date": {
"max": {
"field": "#timestamp"
}
}
}
}
But this causes the error: "failed to parse date field [min_date]"
Is it possible to use the aggregated min and max values to define the date range?
Since you have not provided any sample index data, so applying range query on date type field
Adding a working example with index mapping, data, search query, and search result
Index Mapping:
{
"mappings": {
"properties": {
"date": {
"type": "date"
}
}
}
}
Index Data:
{
"date": "2015-02-10",
"event_id":"1234"
}
{
"date": "2015-01-01",
"event_id":"1235"
}
{
"date": "2015-02-01",
"event_id":"1234"
}
{
"date": "2015-02-01",
"event_id":"1235"
}
{
"date": "2015-01-20",
"event_id":"1234"
}
Search Query:
{
"query": {
"bool": {
"must": {
"match": {
"event_id": 1234
}
},
"filter": {
"range": {
"date": {
"lte": "2015-02-15",
"gte": "2015-01-11"
}
}
}
}
},
"aggs": {
"min_date": {
"min": {
"field": "date"
}
},
"max_date": {
"max": {
"field": "date"
}
}
}
}
Search Result:
"hits": {
"total": {
"value": 3,
"relation": "eq"
},
"max_score": 0.44183272,
"hits": [
{
"_index": "stof_64127765",
"_type": "_doc",
"_id": "3",
"_score": 0.44183272,
"_source": {
"date": "2015-02-01",
"event_id": "1234"
}
},
{
"_index": "stof_64127765",
"_type": "_doc",
"_id": "1",
"_score": 0.44183272,
"_source": {
"date": "2015-02-10",
"event_id": "1234"
}
},
{
"_index": "stof_64127765",
"_type": "_doc",
"_id": "5",
"_score": 0.44183272,
"_source": {
"date": "2015-01-20",
"event_id": "1234"
}
}
]
},
"aggregations": {
"max_date": {
"value": 1.4235264E12,
"value_as_string": "2015-02-10T00:00:00.000Z"
},
"min_date": {
"value": 1.421712E12,
"value_as_string": "2015-01-20T00:00:00.000Z"
}
}

Cloud Firestore subcollection query via REST

I have collection x, each document of x has subcollection y. Each document of y has a time attribute. I can't figure out how to query just that subcollection via REST (I know this feature exists in the SDK). My query so far, which is obviously wrong:
{
"structuredQuery": {
"from": [
{
"collectionId": "x",
"allDescendants": true
}
],
"where": {
"compositeFilter": {
"op": "AND",
"filters": [
{
"fieldFilter": {
"field": {
"fieldPath": "y.time"
},
"op": "GREATER_THAN_OR_EQUAL",
"value": {
"integerValue": 1577836800000
}
}
},
{
"fieldFilter": {
"field": {
"fieldPath": "y.time"
},
"op": "LESS_THAN_OR_EQUAL",
"value": {
"integerValue": 1578355200000
}
}
}
]
}
}
}
}
Sending a POST to https://firestore.googleapis.com/v1/projects/PROJECT/databases/{default}/documents:runQuery, but I've also tried .../documents/x/ID/y:runQuery but that's obviously wrong too.
I believe you described a collection group query for collection group y. In the REST API, this is an allDescendants query on the path projects/PROJECT/databases/(default)/documents (known as the root document):
https://firestore.googleapis.com/v1/projects/PROJECT/databases/(default)/documents:runQuery
{
"structuredQuery": {
"from": [
{
"collectionId": "y",
"allDescendants": true
}
],
"where": {
"compositeFilter": {
"op": "AND",
"filters": [
{
"fieldFilter": {
"field": {
"fieldPath": "time"
},
"op": "GREATER_THAN_OR_EQUAL",
"value": {
"integerValue": 1577836800000
}
}
},
{
"fieldFilter": {
"field": {
"fieldPath": "time"
},
"op": "LESS_THAN_OR_EQUAL",
"value": {
"integerValue": 1578355200000
}
}
}
]
}
}
}
}
Declare the path to the subcollection in the URL:
https://firestore.googleapis.com/v1/projects/PROJECT/databases/(default)/documents/x/documentX:runQuery
Then make the collectionId in from collection "y" and allDescendants false:
{
"structuredQuery": {
"from": [
{
"collectionId": "y",
"allDescendants": false
}
],
"where": {
"compositeFilter": {
"op": "AND",
"filters": [
{
"fieldFilter": {
"field": {
"fieldPath": "y.time"
},
"op": "GREATER_THAN_OR_EQUAL",
"value": {
"integerValue": 1577836800000
}
}
},
{
"fieldFilter": {
"field": {
"fieldPath": "y.time"
},
"op": "LESS_THAN_OR_EQUAL",
"value": {
"integerValue": 1578355200000
}
}
}
]
}
}
}
}
Source: https://firebase.google.com/docs/firestore/reference/rest/v1/projects.databases.documents/runQuery#path-parameters

Match_phrase isn’t precise

this is my REST API:
GET logstash-2017.12.29/_search
{
"_source": {
"includes": [ "IPV4_DST_ADDR","IPV4_SRC_ADDR","IN_BYTES","OUT_BYTES"]
},
"size" : 100,
"query": {
"bool": {
"should": [
{
"match_phrase":{"IPV4_DST_ADDR":"192.168.0.159"}
},
{
"match_phrase":{"IPV4_SRC_ADDR":"192.168.0.159"}
}
],
"must":
{
"range" : {
"LAST_SWITCHED" : {
"gte" : 1514543547
}
}
}
}
},
"aggs": {
"IN_PKTS": {
"sum": {
"field": "IN_PKTS"
}
},
"IN_BYTES": {
"sum": {
"field": "IN_BYTES"
}
},
"OUT_BYTES": {
"sum": {
"field": "OUT_BYTES"
}
},
"OUT_PKTS": {
"sum": {
"field": "OUT_PKTS"
}
},
"genres":{
"terms" : {
"field" : "L7_PROTO_NAME.keyword",
"order" : { "in_bytes" : "desc" }
},
"aggs":{
"in_bytes": {
"sum": { "field":"IN_BYTES"}
}
}
},
"download1" : {
"filter" : { "term": { "IPV4_DST_ADDR":"192.168.0.159"} },
"aggs" : {
"downlod_bytes" : { "sum" : { "field" : "IN_BYTES" } }
}
},
"download2" : {
"filter" : { "term": { "IPV4_SRC_ADDR":"192.168.0.159"} },
"aggs" : {
"downlod_bytes" : { "sum" : { "field" : "OUT_BYTES" } }
}
},"upload1" : {
"filter" : { "term": { "IPV4_DST_ADDR":"192.168.0.159"} },
"aggs" : {
"downlod_bytes" : { "sum" : { "field" : "OUT_BYTES" } }
}
},"upload2" : {
"filter" : { "term": { "IPV4_SRC_ADDR":"192.168.0.159"} },
"aggs" : {
"downlod_bytes" : { "sum" : { "field" : "IN_BYTES" } }
}
}
}
I found there are some return documents didn't meet my requirement.
{
"_index": "logstash-2017.12.29",
"_type": "ntopng-*",
"_id": "AWCh1jPtnZ2m3739FTU7",
"_score": 1,
"_source": {
"IPV4_SRC_ADDR": "192.168.0.109", // not in my expectation
"IN_BYTES": 132,
"IPV4_DST_ADDR": "224.0.0.252", // not in my expectation
"OUT_BYTES": 0
}
}
the return document IPV4_SRC_ADDR or IPV4_DST_ADDR are not "192.168.0.159".
it seems fuzzy search, but I want to match_phrase 100%.
either IPV4_SRC_ADDR or IPV4_DST_ADDR is "192.168.0.159".
How should I modified my REST API .
thank you in advance!
You should map your IP fields using the ip data type
{
"mappings": {
"my_type": {
"properties": {
"IPV4_SRC_ADDR": {
"type": "ip"
},
"IPV4_DST_ADDR": {
"type": "ip"
}
}
}
}
}
Then you'll be able to match those addresses exactly using a simple term query:
"should": [
{
"term":{"IPV4_DST_ADDR":"192.168.0.159"}
},
{
"term":{"IPV4_SRC_ADDR":"192.168.0.159"}
}
],
UPDATE:
Given your mapping you can also use the .keyword sub-field, like this
{
"_source": {
"includes": [
"IPV4_DST_ADDR",
"IPV4_SRC_ADDR",
"IN_BYTES",
"OUT_BYTES"
]
},
"size": 100,
"query": {
"bool": {
"minimum_should_match": 1,
"should": [
{
"term": {
"IPV4_DST_ADDR.keyword": "192.168.0.159"
}
},
{
"term": {
"IPV4_SRC_ADDR.keyword": "192.168.0.159"
}
}
],
"must": {
"range": {
"LAST_SWITCHED": {
"gte": 1514543547
}
}
}
}
},
"aggs": {
"IN_PKTS": {
"sum": {
"field": "IN_PKTS"
}
},
"IN_BYTES": {
"sum": {
"field": "IN_BYTES"
}
},
"OUT_BYTES": {
"sum": {
"field": "OUT_BYTES"
}
},
"OUT_PKTS": {
"sum": {
"field": "OUT_PKTS"
}
},
"genres": {
"terms": {
"field": "L7_PROTO_NAME.keyword",
"order": {
"in_bytes": "desc"
}
},
"aggs": {
"in_bytes": {
"sum": {
"field": "IN_BYTES"
}
}
}
},
"download1": {
"filter": {
"term": {
"IPV4_DST_ADDR.keyword": "192.168.0.159"
}
},
"aggs": {
"download_bytes": {
"sum": {
"field": "IN_BYTES"
}
}
}
},
"download2": {
"filter": {
"term": {
"IPV4_SRC_ADDR.keyword": "192.168.0.159"
}
},
"aggs": {
"downlod_bytes": {
"sum": {
"field": "OUT_BYTES"
}
}
}
},
"upload1": {
"filter": {
"term": {
"IPV4_DST_ADDR.keyword": "192.168.0.159"
}
},
"aggs": {
"downlod_bytes": {
"sum": {
"field": "OUT_BYTES"
}
}
}
},
"upload2": {
"filter": {
"term": {
"IPV4_SRC_ADDR.keyword": "192.168.0.159"
}
},
"aggs": {
"downlod_bytes": {
"sum": {
"field": "IN_BYTES"
}
}
}
}
}
}

Elasticsearch: Date Aggregation Most Recent

I have query that works. It aggregates data based on Id and finds the MOST RECENT object based on the created field. The problem I have is that I would like to find the SECOND MOST RECENT instead of MOST RECENT. How would I go about this? I have been looking all through the docs and all I can find is range which doesn't help me to much. Thank you :)
{
"query":{
"match": {
"name": "Robert"
}
},
"aggs": {
"previous": {
"terms": {
"field": "Id",
"order": {"timeCreated": "desc"}
},
"aggs": {
"timeCreated": {
"max": {"field": "created"}
}
}
}
}
}
Top_hits is what you are looking for. Use this:
{
"query":{
"match": {
"name": "A"
}
},
"aggs": {
"previous": {
"terms": {
"field": "Id"
},
"aggs": {
"latestRecords": {
"top_hits": {
"sort": {
"created": {
"order": "desc"
}
},
"size" :2
}
}
}
}
}
}