Related
We're trying to optimise our read performance on our MongoDB cluster.
We serve a social media like application where users are member of 1 or multiple groups.
We were storing who is in which group and whether he/she is an admin of that group in a separate collection. However we noticed it was quite slow to retrieve the group information for the groups the user is member of. (find(+filter) groupMember documents, populate the groups).
Therefor we recently migrated all the group members to an array on the group collection documents itself.
The schema now looks as following:
The query we execute is simply:
this.model.find({
members: {
$elemMatch: {
userId: new ObjectId(userId),
},
},
})
We expected this to be much more performed because you don't need to populate/lookup anything. The opposite is true however, after deploying this change we noticed a performance decrease.
We have around 40k group documents where the largest groups have around 3k members, most groups are much smaller however.
The groups are indexed and the index is also used. This is an explain plan:
{
"explainVersion": "1",
"queryPlanner": {
"namespace": "***.groups",
"indexFilterSet": false,
"parsedQuery": {
"members": {
"$elemMatch": {
"userId": {
"$eq": "61b091ee9b50220e75208eb6"
}
}
}
},
"queryHash": "DCF50157",
"planCacheKey": "DCF50157",
"maxIndexedOrSolutionsReached": false,
"maxIndexedAndSolutionsReached": false,
"maxScansToExplodeReached": false,
"winningPlan": {
"stage": "FETCH",
"filter": {
"members": {
"$elemMatch": {
"userId": {
"$eq": "61b091ee9b50220e75208eb6"
}
}
}
},
"inputStage": {
"stage": "IXSCAN",
"keyPattern": {
"members.userId": 1
},
"indexName": "members.userId_1",
"isMultiKey": true,
"multiKeyPaths": {
"members.userId": [
"members"
]
},
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2,
"direction": "forward",
"indexBounds": {
"members.userId": [
"[ObjectId('61b091ee9b50220e75208eb6'), ObjectId('61b091ee9b50220e75208eb6')]"
]
}
}
},
"rejectedPlans": []
},
"executionStats": {
"executionSuccess": true,
"nReturned": 17,
"executionTimeMillis": 0,
"totalKeysExamined": 17,
"totalDocsExamined": 17,
"executionStages": {
"stage": "FETCH",
"filter": {
"members": {
"$elemMatch": {
"userId": {
"$eq": "61b091ee9b50220e75208eb6"
}
}
}
},
"nReturned": 17,
"executionTimeMillisEstimate": 0,
"works": 18,
"advanced": 17,
"needTime": 0,
"needYield": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"docsExamined": 17,
"alreadyHasObj": 0,
"inputStage": {
"stage": "IXSCAN",
"nReturned": 17,
"executionTimeMillisEstimate": 0,
"works": 18,
"advanced": 17,
"needTime": 0,
"needYield": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"keyPattern": {
"members.userId": 1
},
"indexName": "members.userId_1",
"isMultiKey": true,
"multiKeyPaths": {
"members.userId": [
"members"
]
},
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2,
"direction": "forward",
"indexBounds": {
"members.userId": [
"[ObjectId('61b091ee9b50220e75208eb6'), ObjectId('61b091ee9b50220e75208eb6')]"
]
},
"keysExamined": 17,
"seeks": 1,
"dupsTested": 17,
"dupsDropped": 0
}
},
"allPlansExecution": []
},
"command": {
"find": "groups",
"filter": {
"members": {
"$elemMatch": {
"userId": "61b091ee9b50220e75208eb6"
}
}
},
"projection": {},
"readConcern": {
"level": "majority"
},
"$db": "***"
},
"serverInfo": {
"host": "***",
"port": 27017,
"version": "6.0.3",
"gitVersion": "f803681c3ae19817d31958965850193de067c516"
},
"serverParameters": {
"internalQueryFacetBufferSizeBytes": 104857600,
"internalQueryFacetMaxOutputDocSizeBytes": 104857600,
"internalLookupStageIntermediateDocumentMaxSizeBytes": 104857600,
"internalDocumentSourceGroupMaxMemoryBytes": 104857600,
"internalQueryMaxBlockingSortMemoryUsageBytes": 104857600,
"internalQueryProhibitBlockingMergeOnMongoS": 0,
"internalQueryMaxAddToSetBytes": 104857600,
"internalDocumentSourceSetWindowFieldsMaxMemoryBytes": 104857600
},
"ok": 1,
"operationTime": {
"$timestamp": "7168789227251957761"
}
}
Under load the query takes 300-400ms, which is not acceptable for us.
However right now we don't really know anymore what would be the best next step in improving the solution. Mongo does not advise any additional indexes or schema improvements at this moment.
What can we do best to get this query really performand?
I have the below dbData wherein in the contracts collection, there is an attribute called via which is an array that holds the mongoose ids of depot entries.
db={
"contracts": [
{
"_id": ObjectId("630c9e23fb146c4b3b801b9e"),
"detention_id": [],
"route_exclude": false,
"trip_count": null,
"fill_first": false,
"is_dedicated": false,
"truck_count": 0,
"running_trip_count": 0,
"is_round_trip_eligible": false,
"is_synced": false,
"bdr_label": "",
"bdr_charge": 0,
"is_migrated": false,
"current_trip_count": 0,
"client_id": ObjectId("5e47c1f4a697c60195050059"),
"updated_by": ObjectId("60a39adad079f9004d144cb8"),
"user_id": ObjectId("60a39adad079f9004d144cb8"),
"transporter_id": ObjectId("61d6d2227465c501d0915794"),
"vehicle_type_id": ObjectId("61e1b3ad00516e004c893145"),
"detention_group": "",
"delivery_type": 2,
"source": "amaravathi",
"destination": "Kozhikode",
"isDirect": false,
"via": [
"6304804a9f90b8018844fc20",
"62fb93a711ea3f019ffe19e6"
],
"distance": 100,
"status": 1,
"contract_type": 1,
"current": {
"minimum_rate": 0,
"multiplying_factor": null,
"min_slab": 0,
"additional_charge_percentage": 0,
"hilly_region_addition_percentage": 0,
"return_freight_rate": 0,
"surge_threshold": 0,
"fixed_rate": null,
"oda_rate": null,
"oda_unit": 3,
"minimum_oda": 0,
"point_charges": 0,
"pickup_point_charges": 0,
"start": ISODate("2022-08-26T00:00:00.000+05:30"),
"end": ISODate("2023-08-24T23:59:00.000+05:30"),
"freight_unit": 1,
"freight_rate": 5000,
"transit_time": 172800000
},
"next": [],
"previous": [],
"lookup_key": "f93187d8364fca313ab92c83ac39eafa",
"created_at": ISODate("2022-08-26T15:26:15.865+05:30"),
"updated_at": ISODate("2022-08-29T13:20:39.058+05:30")
}
],
"depots": [
{
"_id": ObjectId("6304804a9f90b8018844fc20"),
"consignee_ref_ids": [],
"status": 1,
"coordinates": [
12.9039441,
80.14567129999999
],
"unloading_capacity": 0,
"has_pit_stop": false,
"type": 1,
"plant_type": 1,
"ports_associated": [],
"overseas_customer": false,
"sez": false,
"timezone": "Asia/Kolkata",
"fax_number": "",
"is_optima_group_by_cluster": "",
"country": "India",
"name": "CTB",
"reference_id": "CTB",
"client_id": ObjectId("5e47c1f4a697c60195050059"),
"address": "CTB",
"city": "Chennai",
"state": "Tamil Nadu",
"region": "South",
"pincode": "600126",
"phone": "",
"delivery_type": 1,
"short_code": "CTB",
"gstn_no": "HSKK309393k",
"gates": [
{
"coordinates": [
12.9039441,
80.14567129999999
],
"exclude_trucks": [],
"_id": ObjectId("630480789f90b8018844fc21"),
"country": "India",
"name": "CTG1",
"reference_id": "CTG1",
"status": 1,
"address": "CTB",
"city": "Chennai",
"state": "Tamil Nadu",
"region": "South",
"pincode": "600126",
"short_code": "CTG1",
"special_tax_rebate": false,
"gstin": null,
"is_virtual_gate": false,
"category": "",
"delivery_do_not_combine": false,
"combine": false,
"loading_time": {
"start": null,
"end": null
},
"unloading_time": {
"start": null,
"end": null
},
"legal_entity": null,
"display_reference_name": "",
"updated_at": ISODate("2022-08-23T12:53:46.233+05:30"),
"created_at": ISODate("2022-08-23T12:53:36.046+05:30")
}
],
"docks": [],
"updated_by": ObjectId("5eaba6438a4895004916adbe"),
"display_reference_name": "",
"receivers": [],
"created_at": ISODate("2022-08-23T12:52:50.182+05:30"),
"updated_at": ISODate("2022-08-23T13:09:18.468+05:30")
},
{
"_id": ObjectId("62fb93a711ea3f019ffe19e6"),
"consignee_ref_ids": [],
"status": 1,
"coordinates": [
20.593684,
78.96288
],
"unloading_capacity": 0,
"has_pit_stop": false,
"type": 1,
"plant_type": 1,
"ports_associated": [],
"overseas_customer": false,
"sez": false,
"timezone": "Asia/Kolkata",
"fax_number": "",
"is_optima_group_by_cluster": "",
"country": "India",
"name": "Test",
"reference_id": "Test",
"client_id": ObjectId("5e47c1f4a697c60195050059"),
"address": "test",
"city": "Test",
"state": "Test",
"region": "North",
"pincode": "0100301",
"phone": "",
"delivery_type": 1,
"short_code": "test",
"gstn_no": "1111",
"gates": [],
"docks": [],
"updated_by": ObjectId("62e7b04d91a9b50189b4319d"),
"display_reference_name": "",
"receivers": [],
"created_at": ISODate("2022-08-16T18:25:03.453+05:30"),
"updated_at": ISODate("2022-08-16T18:25:03.913+05:30")
},
{
"_id": ObjectId("630cc2ba38f87401899c35f1"),
"consignee_ref_ids": [],
"status": 1,
"coordinates": [
20.593684,
78.96288
],
"unloading_capacity": 0,
"has_pit_stop": false,
"type": 1,
"plant_type": 1,
"ports_associated": [],
"overseas_customer": false,
"sez": false,
"timezone": "Asia/Kolkata",
"fax_number": "",
"is_optima_group_by_cluster": "",
"country": "India",
"name": "B20220829191440",
"reference_id": "B20220829191440",
"client_id": ObjectId("62d8f30ce763689b942d5e12"),
"address": "No 15,GST Street",
"city": "Blr20220829191440",
"state": "Karnataka",
"region": "South",
"pincode": "123456",
"phone": "5588",
"delivery_type": 1,
"short_code": "B214945",
"gstn_no": "123",
"gates": [
{
"coordinates": [
10.9290666,
78.7438114
],
"exclude_trucks": [],
"_id": ObjectId("630cc2bb38f87401899c35f2"),
"country": "India",
"name": "Gate 2",
"reference_id": "BG20220829191440",
"status": 1,
"address": "No 15 GST Road",
"city": "Blr20220829191440",
"state": "Karnataka",
"region": "South",
"pincode": "123456",
"short_code": "G1214945",
"special_tax_rebate": false,
"is_virtual_gate": false,
"category": "b",
"combine": true,
"updated_at": ISODate("2022-08-29T19:14:27.912+05:30"),
"created_at": ISODate("2022-08-29T19:14:27.551+05:30")
}
],
"docks": [],
"updated_by": ObjectId("62d8f30ce763689b942d5e15"),
"display_reference_name": "",
"receivers": [],
"created_at": ISODate("2022-08-29T19:14:26.892+05:30"),
"updated_at": ISODate("2022-08-29T19:14:27.912+05:30")
}
]
}
I am trying to aggregate using lookup such that I need to fetch the entries from the depots based on the ids that are present in via.
Below is the query I have tried
db.contracts.aggregate([
{
$match: {
_id: ObjectId("630c9e23fb146c4b3b801b9e")
}
},
{
$lookup: {
from: "depots",
localField: "via",
as: "depots",
foreignField: "_id"
}
},
{
$sort: {
count: -1
}
}
])
for which I expect to get the contracts entry with a newly added attribute in it called depots that contains the documents for the corressponding two ids held by via.
But the actual response that I got has depots as [].
[
{
"_id": ObjectId("630c9e23fb146c4b3b801b9e"),
"bdr_charge": 0,
"bdr_label": "",
"client_id": ObjectId("5e47c1f4a697c60195050059"),
"contract_type": 1,
"created_at": ISODate("2022-08-26T09:56:15.865Z"),
"current": {
"additional_charge_percentage": 0,
"end": ISODate("2023-08-24T18:29:00Z"),
"fixed_rate": null,
"freight_rate": 5000,
"freight_unit": 1,
"hilly_region_addition_percentage": 0,
"min_slab": 0,
"minimum_oda": 0,
"minimum_rate": 0,
"multiplying_factor": null,
"oda_rate": null,
"oda_unit": 3,
"pickup_point_charges": 0,
"point_charges": 0,
"return_freight_rate": 0,
"start": ISODate("2022-08-25T18:30:00Z"),
"surge_threshold": 0,
"transit_time": 1.728e+08
},
"current_trip_count": 0,
"delivery_type": 2,
"depots": [],
"destination": "Kozhikode",
"detention_group": "",
"detention_id": [],
"distance": 100,
"fill_first": false,
"isDirect": false,
"is_dedicated": false,
"is_migrated": false,
"is_round_trip_eligible": false,
"is_synced": false,
"lookup_key": "f93187d8364fca313ab92c83ac39eafa",
"next": [],
"previous": [],
"route_exclude": false,
"running_trip_count": 0,
"source": "amaravathi",
"status": 1,
"transporter_id": ObjectId("61d6d2227465c501d0915794"),
"trip_count": null,
"truck_count": 0,
"updated_at": ISODate("2022-08-29T07:50:39.058Z"),
"updated_by": ObjectId("60a39adad079f9004d144cb8"),
"user_id": ObjectId("60a39adad079f9004d144cb8"),
"vehicle_type_id": ObjectId("61e1b3ad00516e004c893145"),
"via": [
"6304804a9f90b8018844fc20",
"62fb93a711ea3f019ffe19e6"
]
}
]
Question 1: Could some help me on where Iam going wrong with the $lookup implementation?
Question 2: Also could the same query, be achievable using graphlookup? Atleast with a possible schema change in contracts?
Edit:
when I kept it as objectIds in via instead of strings it works as expected
"via": [
ObjectId("6304804a9f90b8018844fc20"),
ObjectId("62fb93a711ea3f019ffe19e6")
]
When I kept the via as strings, added an additional stage in the pipeline before the $lookup stage to make it work
db.contracts.aggregate([
{
$match: {
_id: ObjectId("630c9e23fb146c4b3b801b9e")
}
},
{
$addFields: {
via: {
$map: {
input: "$via",
in: {
"$toObjectId": "$$this"
}
}
}
}
},
{
$lookup: {
from: "depots",
localField: "via",
as: "depots",
foreignField: "_id"
}
},
{
$sort: {
count: -1
}
}
])
One option is to use a $map stage before the $lookup:
db.contracts.aggregate([
{$match: {_id: ObjectId("630c9e23fb146c4b3b801b9e")}},
{$set: {via: {
$map: {
input: "$via",
in: {"$toObjectId": "$$this"}
}
}
}
},
{$lookup: {
from: "depots",
localField: "via",
as: "depots",
foreignField: "_id"
}
},
{$sort: {count: -1}}
])
See how it works on the playground example
No need to use $graphlookup as you only have one item calling the other and not a recursive call. A relevant alternative to $lookup here will be populate. You can read about this option a bit here
Was able to implement $lookup as well $graphLookup with the below query
db.contracts.aggregate([
{
$match: {
_id: ObjectId("630c9e23fb146c4b3b801b9e")
}
},
{
$addFields: {
via: {
$map: {
input: "$via",
in: {
"$toObjectId": "$$this"
}
}
}
}
},
{
$lookup: {
from: "depots",
localField: "via",
as: "depots",
foreignField: "_id"
}
},
// GraphLookup stage
/* {
"$graphLookup": {
"from": "depots",
"startWith": "$via",
"connectFromField": "via",
"connectToField": "_id",
"as": "depots",
"maxDepth": 2
}
}, */
{
$sort: {
count: -1
}
}
])
I'm trying to create a ConfigMap with ArgoCD.
I've created a volumes.yaml file as such
---
apiVersion: v1
kind: ConfigMap
metadata:
name: persistent-volumes-argo
labels:
grafana_dashboard: "1"
project: "foo"
data:
kubernetes.json: |
{{ .Files.Get "dashboards/persistent-volumes.json" | indent 4 }}
But ArgoCD doesn't seem to be able to read the data, the way a standard Helm deployment would.
I've tried adding the data directly into the ConfigMap as such
(Data omitted for brevity)
---
apiVersion: v1
kind: ConfigMap
metadata:
name: persistent-volumes-argo
labels:
grafana_dashboard: "1"
project: "foo"
data:
kubernetes.json: |
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"limit": 100,
"name": "Annotations & Alerts",
"showIn": 0,
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": 13646,
"graphTooltip": 0,
"iteration": 1659421503107,
"links": [],
"panels": [
{
"collapsed": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 26,
"panels": [],
"title": "Alerts",
"type": "row"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"noValue": "--",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "semi-dark-red",
"value": null
},
{
"color": "light-green",
"value": -0.0001
},
{
"color": "semi-dark-red",
"value": 0.0001
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 8,
"x": 0,
"y": 1
},
"id": 21,
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.0.3",
"targets": [
{
"expr": "count (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"} ) and (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"} )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes{namespace=~\"${k8s_namespace}\"} )) >= (${warning_threshold} / 100)) or vector (0)",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "PVCs Above Warning Threshold",
"type": "stat"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 0,
"mappings": [],
"noValue": "--",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "semi-dark-red",
"value": null
},
{
"color": "light-green",
"value": -0.0001
},
{
"color": "semi-dark-red",
"value": 0.0001
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 8,
"x": 8,
"y": 1
},
"id": 24,
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.0.3",
"targets": [
{
"expr": "count((kube_persistentvolumeclaim_status_phase{namespace=~\"${k8s_namespace}\",phase=\"Pending\"}==1)) or vector(0)",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "PVCs in Pending State",
"transformations": [
{
"id": "organize",
"options": {}
}
],
"type": "stat"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 0,
"mappings": [],
"noValue": "--",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "semi-dark-red",
"value": null
},
{
"color": "light-green",
"value": -0.0001
},
{
"color": "semi-dark-red",
"value": 0.0001
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 8,
"x": 16,
"y": 1
},
"id": 23,
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.0.3",
"targets": [
{
"expr": "count((kube_persistentvolumeclaim_status_phase{namespace=~\"${k8s_namespace}\",phase=\"Lost\"}==1)) or vector(0)",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "PVCs in Lost State",
"transformations": [
{
"id": "organize",
"options": {}
}
],
"type": "stat"
},
{
"collapsed": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 17,
"panels": [],
"title": "Usage statistics",
"type": "row"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": null,
"displayMode": "auto",
"filterable": false
},
"mappings": [],
"noValue": "--",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-green",
"value": null
}
]
},
"unit": "none"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Used (%)"
},
"properties": [
{
"id": "custom.displayMode",
"value": "gradient-gauge"
},
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "light-green",
"value": null
},
{
"color": "semi-dark-yellow",
"value": 70
},
{
"color": "dark-red",
"value": 80
}
]
}
},
{
"id": "decimals",
"value": 1
}
]
},
{
"matcher": {
"id": "byName",
"options": "Status"
},
"properties": [
{
"id": "custom.displayMode",
"value": "color-background"
},
{
"id": "mappings",
"value": [
{
"options": {
"0": {
"text": "Bound"
},
"1": {
"text": "Pending"
},
"2": {
"text": "Lost"
}
},
"type": "value"
}
]
},
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "light-green",
"value": null
},
{
"color": "light-green",
"value": 0
},
{
"color": "semi-dark-orange",
"value": 1
},
{
"color": "semi-dark-red",
"value": 2
}
]
}
},
{
"id": "noValue",
"value": "--"
},
{
"id": "custom.align",
"value": "center"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Namespace"
},
"properties": [
{
"id": "custom.width",
"value": 120
}
]
},
{
"matcher": {
"id": "byName",
"options": "Status"
},
"properties": [
{
"id": "custom.width",
"value": 80
}
]
},
{
"matcher": {
"id": "byName",
"options": "Capacity (GiB)"
},
"properties": [
{
"id": "custom.width",
"value": 120
}
]
},
{
"matcher": {
"id": "byName",
"options": "Used (GiB)"
},
"properties": [
{
"id": "custom.width",
"value": 120
}
]
},
{
"matcher": {
"id": "byName",
"options": "Available (GiB)"
},
"properties": [
{
"id": "custom.width",
"value": 120
}
]
},
{
"matcher": {
"id": "byName",
"options": "StorageClass"
},
"properties": [
{
"id": "custom.width",
"value": 150
}
]
},
{
"matcher": {
"id": "byName",
"options": "PersistentVolumeClaim"
},
"properties": [
{
"id": "custom.width",
"value": 370
}
]
}
]
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 6
},
"id": 29,
"interval": "",
"options": {
"frameIndex": 2,
"showHeader": true,
"sortBy": [
{
"desc": false,
"displayName": "PersistentVolumeClaim"
}
]
},
"pluginVersion": "8.0.3",
"targets": [
{
"expr": " sum by (persistentvolumeclaim,namespace,storageclass,volumename) (kube_persistentvolumeclaim_info{namespace=~\"${k8s_namespace}\"})",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
},
{
"expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"${k8s_namespace}\"}/1024/1024/1024)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "B"
},
{
"expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"}/1024/1024/1024)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "C"
},
{
"expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_available_bytes{namespace=~\"${k8s_namespace}\"}/1024/1024/1024)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "D"
},
{
"expr": "sum(kube_persistentvolumeclaim_status_phase{namespace=~\"${k8s_namespace}\",phase=~\"(Pending|Lost)\"}) by (persistentvolumeclaim) + sum(kube_persistentvolumeclaim_status_phase{namespace=~\"${k8s_namespace}\",phase=~\"(Lost)\"}) by (persistentvolumeclaim)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "E"
},
{
"expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"}/kubelet_volume_stats_capacity_bytes{namespace=~\"${k8s_namespace}\"} * 100)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "F"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Persistent Volume Claim",
"transformations": [
{
"id": "seriesToColumns",
"options": {
"byField": "persistentvolumeclaim"
}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Time 1": true,
"Time 2": true,
"Time 3": true,
"Time 4": true,
"Time 5": true,
"Time 6": true,
"Value #A": true
},
"indexByName": {},
"renameByName": {
"Time 1": "",
"Time 2": "",
"Time 3": "",
"Time 4": "",
"Time 5": "",
"Time 6": "",
"Value #A": "",
"Value #B": "Capacity (GiB)",
"Value #C": "Used (GiB)",
"Value #D": "Available (GiB)",
"Value #E": "Status",
"Value #F": "Used (%)",
"namespace": "Namespace",
"persistentvolumeclaim": "PersistentVolumeClaim",
"storageclass": "StorageClass",
"volumename": "PhysicalVolume"
}
}
}
],
"type": "table"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {
"align": null,
"displayMode": "auto",
"filterable": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 18
},
"id": 7,
"options": {
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "Status"
}
]
},
"pluginVersion": "8.0.3",
"targets": [
{
"expr": "kube_storageclass_info",
"format": "table",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Storage Class",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Value": true,
"__name__": true,
"app_kubernetes_io_instance": true,
"app_kubernetes_io_name": true,
"instance": true,
"job": true,
"kubernetes_namespace": true,
"kubernetes_pod_name": true,
"pod_template_hash": true
},
"indexByName": {
"Time": 1,
"Value": 13,
"__name__": 2,
"app_kubernetes_io_instance": 3,
"app_kubernetes_io_name": 4,
"instance": 5,
"job": 6,
"kubernetes_namespace": 7,
"kubernetes_pod_name": 8,
"pod_template_hash": 9,
"provisioner": 10,
"reclaimPolicy": 11,
"storageclass": 0,
"volumeBindingMode": 12
},
"renameByName": {
"provisioner": "Provisioner",
"reclaimPolicy": "ReclaimPolicy",
"storageclass": "StorageClass",
"volumeBindingMode": "VolumeBindingMode"
}
}
},
{
"id": "groupBy",
"options": {
"fields": {
"Provisioner": {
"aggregations": [],
"operation": "groupby"
},
"ReclaimPolicy": {
"aggregations": [],
"operation": "groupby"
},
"StorageClass": {
"aggregations": [],
"operation": "groupby"
},
"VolumeBindingMode": {
"aggregations": [],
"operation": "groupby"
}
}
}
}
],
"type": "table"
},
{
"collapsed": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 23
},
"id": 15,
"panels": [],
"title": "Graphical usage data ",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 24
},
"hiddenSeries": false,
"id": 9,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": true,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.0.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"}))",
"interval": "",
"legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "All Running PVCs Used Bytes",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "Date & time",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": true,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 36
},
"id": 19,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 41
},
"hiddenSeries": false,
"id": 11,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.2.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"}[1h])",
"instant": false,
"interval": "",
"legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Hourly Volume Usage Rate",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "binBps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "Date & time",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 48
},
"hiddenSeries": false,
"id": 12,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.2.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"}[1d])",
"interval": "",
"legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Daily Volume Usage Rate",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "binBps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "Date & time",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 55
},
"hiddenSeries": false,
"id": 13,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
}
}
But this errors with rpc error: code = FailedPrecondition desc = Failed to unmarshal "volumes.yaml": <nil>
Is there a way to pass in json data when creating a ConfigMap with ArgoCD, either as a template or by dumping the data in the file?
To create configmap with argocd and helm
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "helm-chart.fullname" . }}-configmap
data:
config.json : |
{{ toJson .Values.configmap | indent 4 }}
and the value file should be like this, which is our JSON but converted to YAML
configmap:
json:
- rigid
- better for data interchange
yaml:
- slim and flexible
- better for configuration
object:
key: value
array:
- null_value:
- boolean: true
- integer: 1
- alias: &example aliases are like variables
- alias: *example
paragraph: >
Blank lines denote
paragraph breaks
content: |-
Or we
can auto
convert line breaks
to save space
alias: &foo
bar: baz
alias_reuse: *foo
json2yaml
A complete demo app can be found here
A very quick way to create app with the above configmap and demo app
argocd app create demo-app --repo https://github.com/Adiii717/argocd-demo-app.git --path helm-chart --dest-namespace default --dest-server https://kubernetes.default.svc --helm-set replicaCount=2
Make sure you also set
export ARGOCD_AUTH_TOKEN="tokeh
export ARGOCD_OPTS="--grpc-web"
export ARGOCD_SERVER="argocd.example.com"
added section for the configmap
I am trying to build a Python script using PyMongo that will be able to hit a Mongo DB that can get exact matches of n number of objects that may exist in the database. Currently, I have this setup:
db.entries.find({'$or': [<list-of-objects]})
Where the list of objects looks something like this:
[{'email': 'some#email.com', 'zip': '11111'}, {'email': 'another#email.com', 'zip': '11112'}, ...]
Using $or works okay when I have 10 or so items in the list. I am testing now with 100, and it is taking a very long time to return. I have considered using multiple $in filters, but I don't know if that is the best option.
I'm sure there is a better way to handle this, but I am fairly new to Mongo.
EDIT: Output of .explain() below:
{
"executionStats": {
"executionTimeMillis": 228734,
"nReturned": 2,
"totalKeysExamined": 0,
"allPlansExecution": [],
"executionSuccess": true,
"executionStages": {
"needYield": 0,
"saveState": 43556,
"restoreState": 43556,
"isEOF": 1,
"inputStage": {
"needYield": 0,
"saveState": 43556,
"restoreState": 43556,
"isEOF": 1,
"inputStage": {
"needYield": 0,
"direction": "forward",
"saveState": 43556,
"restoreState": 43556,
"isEOF": 1,
"docsExamined": 5453000,
"nReturned": 2,
"needTime": 5452999,
"filter": {
"$or": [{
"$and": [{
"email": {
"$eq": "some#email.com"
}
}, {
"zipcode": {
"$eq": "11111"
}
}]
}, {
"$and": [{
"email": {
"$eq": "another#email.com"
}
}, {
"zipcode": {
"$eq": "11112"
}
}]
}]
},
"executionTimeMillisEstimate": 208083,
"invalidates": 0,
"works": 5453002,
"advanced": 2,
"stage": "COLLSCAN"
},
"nReturned": 2,
"needTime": 5452999,
"executionTimeMillisEstimate": 211503,
"transformBy": {
"_id": false
},
"invalidates": 0,
"works": 5453002,
"advanced": 2,
"stage": "PROJECTION"
},
"nReturned": 2,
"needTime": 5452999,
"executionTimeMillisEstimate": 213671,
"invalidates": 0,
"works": 5453002,
"advanced": 2,
"stage": "SUBPLAN"
},
"totalDocsExamined": 5453000
},
"queryPlanner": {
"parsedQuery": {
"$or": [{
"$and": [{
"email": {
"$eq": "some#email.com"
}
}, {
"zipcode": {
"$eq": "11111"
}
}]
}, {
"$and": [{
"email": {
"$eq": "another#email.com"
}
}, {
"zipcode": {
"$eq": "11112"
}
}]
}]
},
"rejectedPlans": [],
"namespace": "db.entries",
"winningPlan": {
"inputStage": {
"transformBy": {
"_id": false
},
"inputStage": {
"filter": {
"$or": [{
"$and": [{
"email": {
"$eq": "some#email.com"
}
}, {
"zipcode": {
"$eq": "11111"
}
}]
}, {
"$and": [{
"email": {
"$eq": "another#email.com"
}
}, {
"zipcode": {
"$eq": "11112"
}
}]
}]
},
"direction": "forward",
"stage": "COLLSCAN"
},
"stage": "PROJECTION"
},
"stage": "SUBPLAN"
},
"indexFilterSet": false,
"plannerVersion": 1
},
"ok": 1.0,
"serverInfo": {
"host": "somehost",
"version": "3.4.6",
"port": 27017,
"gitVersion": "c55eb86ef46ee7aede3b1e2a5d184a7df4bfb5b5"
}
}
I suggest to create a new index (a compound index) as in your case you are using two fields for your search:
db.entries.createIndex( {"email": 1, "zip": 1} )
Now run your query appending the explain() command at your query, you should see that instead of COLLSCAN it has started to use IXSCAN.
To avoid indexing and re-indexing (this query will not just pertain to email/zip, would be dynamic), I build the lists of data with each header and use them as $in arguments, then pass those into an $and. It seems to be working well enough and it hasn't queried longer than 3 minutes.
Example:
{'$and': [{'email': {'$in': ['some#example.com', 'fake#example.com', 'email#example.com']}, 'zipcode': {'$in': ['12345', '11111', '11112']}}]}
I need make a query with indice at the mongodb, I will show below a minimal case example of my real case.
I have following collection with flowing data:
devsrv(mongod-3.0.4) test> db.teste.find()
{
"_id": ObjectId("57b324c341aaa4b930ef3b92"),
"a": 1,
"b": 1
}
{
"_id": ObjectId("57b324c941aaa4b930ef3b93"),
"a": 1,
"b": 2
}
{
"_id": ObjectId("57b324cd41aaa4b930ef3b94"),
"a": 1,
"b": 3
}
{
"_id": ObjectId("57b324d141aaa4b930ef3b95"),
"a": 1,
"b": 4
}
{
"_id": ObjectId("57b324d541aaa4b930ef3b96"),
"a": 1,
"b": 5
}
{
"_id": ObjectId("57b324da41aaa4b930ef3b97"),
"a": 1,
"b": 6
}
{
"_id": ObjectId("57b324df41aaa4b930ef3b98"),
"a": 1,
"b": 7
}
{
"_id": ObjectId("57b324e441aaa4b930ef3b99"),
"a": 1,
"b": 8
}
{
"_id": ObjectId("57b324f341aaa4b930ef3b9a"),
"a": 1,
"b": ""
}
{
"_id": ObjectId("57b324f641aaa4b930ef3b9b"),
"a": 1,
"b": " "
}
{
"_id": ObjectId("57b324fc41aaa4b930ef3b9c"),
"a": 1,
"b": null
}
{
"_id": ObjectId("57b3250341aaa4b930ef3b9d"),
"a": 1
}
{
"_id": ObjectId("57b46ace41aaa4b930ef3b9e"),
"a": 2
}
And I have the following indexes:
devsrv(mongod-3.0.4) test> db.teste.getIndexes()
[
{
"v": 1,
"key": {
"_id": 1
},
"name": "_id_",
"ns": "test.teste"
},
{
"v": 1,
"key": {
"a": 1,
"b": 1
},
"name": "a_1_b_1",
"ns": "test.teste"
},
{
"v": 1,
"key": {
"b": 1
},
"name": "b_1",
"ns": "test.teste"
}
]
And I need make a query equal this:
devsrv(mongod-3.0.4) test> db.teste.find({$or:[{"b":null},{"b":""},{"b":" "},{"b":{$lt:3}}],"a":1}).explain("executionStats")
{
"queryPlanner": {
"plannerVersion": 1,
"namespace": "test.teste",
"indexFilterSet": false,
"parsedQuery": {
"$and": [
{
"$or": [
{
"b": {
"$eq": null
}
},
{
"b": {
"$eq": ""
}
},
{
"b": {
"$eq": " "
}
},
{
"b": {
"$lt": 3
}
}
]
},
{
"a": {
"$eq": 1
}
}
]
},
"winningPlan": {
"stage": "FETCH",
"filter": {
"a": {
"$eq": 1
}
},
"inputStage": {
"stage": "FETCH",
"filter": {
"$or": [
{
"b": {
"$eq": null
}
},
{
"b": {
"$eq": ""
}
},
{
"b": {
"$eq": " "
}
},
{
"b": {
"$lt": 3
}
}
]
},
"inputStage": {
"stage": "IXSCAN",
"keyPattern": {
"b": 1
},
"indexName": "b_1",
"isMultiKey": false,
"direction": "forward",
"indexBounds": {
"b": [
"[null, null]",
"[-inf.0, 3.0)",
"[\"\", \"\"]",
"[\" \", \" \"]"
]
}
}
}
},
"rejectedPlans": [
{
"stage": "FETCH",
"filter": {
"$or": [
{
"b": {
"$eq": null
}
},
{
"b": {
"$eq": ""
}
},
{
"b": {
"$eq": " "
}
},
{
"b": {
"$lt": 3
}
}
]
},
"inputStage": {
"stage": "IXSCAN",
"keyPattern": {
"a": 1,
"b": 1
},
"indexName": "a_1_b_1",
"isMultiKey": false,
"direction": "forward",
"indexBounds": {
"a": [
"[1.0, 1.0]"
],
"b": [
"[MinKey, MaxKey]"
]
}
}
}
]
},
"executionStats": {
"executionSuccess": true,
"nReturned": 6,
"executionTimeMillis": 0,
"totalKeysExamined": 8,
"totalDocsExamined": 14,
"executionStages": {
"stage": "FETCH",
"filter": {
"a": {
"$eq": 1
}
},
"nReturned": 6,
"executionTimeMillisEstimate": 0,
"works": 10,
"advanced": 6,
"needTime": 2,
"needFetch": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"invalidates": 0,
"docsExamined": 7,
"alreadyHasObj": 7,
"inputStage": {
"stage": "FETCH",
"filter": {
"$or": [
{
"b": {
"$eq": null
}
},
{
"b": {
"$eq": ""
}
},
{
"b": {
"$eq": " "
}
},
{
"b": {
"$lt": 3
}
}
]
},
"nReturned": 7,
"executionTimeMillisEstimate": 0,
"works": 8,
"advanced": 7,
"needTime": 1,
"needFetch": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"invalidates": 0,
"docsExamined": 7,
"alreadyHasObj": 0,
"inputStage": {
"stage": "IXSCAN",
"nReturned": 7,
"executionTimeMillisEstimate": 0,
"works": 8,
"advanced": 7,
"needTime": 1,
"needFetch": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"invalidates": 0,
"keyPattern": {
"b": 1
},
"indexName": "b_1",
"isMultiKey": false,
"direction": "forward",
"indexBounds": {
"b": [
"[null, null]",
"[-inf.0, 3.0)",
"[\"\", \"\"]",
"[\" \", \" \"]"
]
},
"keysExamined": 8,
"dupsTested": 0,
"dupsDropped": 0,
"seenInvalidated": 0,
"matchTested": 0
}
}
}
},
"serverInfo": {
"host": "devsrv",
"port": 27017,
"version": "3.0.4",
"gitVersion": "0481c958daeb2969800511e7475dc66986fa9ed5"
},
"ok": 1
}
But MongoDB isn't using the two indexes together.
Each $or term is effectively a separate query, so it helps to structure your query so that each term aligns with the index you're hoping to use. In this case that means moving the a: 1 part inside of each $or term:
db.teste.find({
$or:[
{a: 1, b: null},
{a: 1, b: ""},
{a: 1, b: " "},
{a: 1, b: {$lt: 3}}
]}).explain('executionStats')
The explain output shows that the a_1_b_1 is used for this query.
But you can simplify this a bit more by using $in to combine the first three terms into one:
db.teste.find({
$or:[
{a: 1, b: {$in: [null, "", " "]}},
{a: 1, b: {$lt: 3}}
]}).explain('executionStats')
This is also able to use the a_1_b_1 index.
The code is
db.teste.explain("executionStats").find({a: 1,
$or:[{b: null},
{b: ""},
{b: " "},
{b: {$lt:3}}]
}).hint({a: 1, b: 1})
Be careful with the hint command as the query optimizer chooses the most efficient query by measuring actual performance of the query with every suitable index.