while consuming from kafka in druid, roll up merges two rows to 1 instead of adding them - druid

I trying to use druid to consume events from kafka, however when I'm using roll-up to consume the data, the number of events seem to coming in wrong. without roll-up the numbers are accurate. I am using Druid 0.17.1.
i have observed that while roll up is happening instead of aggregating the events to n it aggregates to 1.
here is my ingestion spec
{
"dataSchema": {
"dataSource": "notificationstatus",
"timestampSpec": {
"column": "date",
"format": "yyyy-MM-dd-HH:mm:ss Z",
"missingValue": null
},
"dimensionsSpec": {
"dimensions": [{
"type": "string",
"name": "Process",
"multiValueHandling": "SORTED_ARRAY",
"createBitmapIndex": true
},
{
"type": "string",
"name": "Channel",
"multiValueHandling": "SORTED_ARRAY",
"createBitmapIndex": true
},
{
"type": "string",
"name": "Status",
"multiValueHandling": "SORTED_ARRAY",
"createBitmapIndex": true
},
{
"type": "string",
"name": "Message",
"multiValueHandling": "SORTED_ARRAY",
"createBitmapIndex": true
},
{
"type": "string",
"name": "CampaignID",
"multiValueHandling": "SORTED_ARRAY",
"createBitmapIndex": true
},
{
"type": "string",
"name": "BannerID",
"multiValueHandling": "SORTED_ARRAY",
"createBitmapIndex": true
}
],
"dimensionExclusions": [
"date",
"count"
]
},
"metricsSpec": [{
"type": "count",
"name": "count"
}],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "HOUR",
"queryGranularity": "MINUTE",
"rollup": true,
"intervals": null
},
"transformSpec": {
"filter": {
"type": "not",
"field": {
"type": "like",
"dimension": "Status",
"pattern": "INFO",
"escape": null,
"extractionFn": null
}
},
"transforms": []
}
},
"ioConfig": {
"topic": "notificationstatus",
"inputFormat": {
"type": "tsv",
"columns": [
"source",
"ymd",
"date",
"Process",
"deviceID",
"Channel",
"CampaignID",
"BannerID",
"Status",
"Message",
"11",
"12"
],
"listDelimiter": null,
"delimiter": "\t",
"findColumnsFromHeader": false,
"skipHeaderRows": 0
},
"replicas": 1,
"taskCount": 1,
"taskDuration": "PT3600S",
"consumerProperties": {},
"pollTimeout": 100,
"startDelay": "PT5S",
"period": "PT30S",
"useEarliestOffset": false,
"completionTimeout": "PT1800S",
"lateMessageRejectionPeriod": null,
"earlyMessageRejectionPeriod": null,
"lateMessageRejectionStartDateTime": null,
"stream": "notificationstatus",
"useEarliestSequenceNumber": false,
"type": "kafka"
},
"tuningConfig": {
"type": "kafka",
"maxRowsInMemory": 1000000,
"maxBytesInMemory": 0,
"maxRowsPerSegment": 5000000,
"maxTotalRows": null,
"intermediatePersistPeriod": "PT10M",
"basePersistDirectory": "/home/akash/Downloads/druidVer/apache-druid-0.17.1/var/tmp/druid-realtime-persist622909873559398926",
"maxPendingPersists": 0,
"indexSpec": {
"bitmap": {
"type": "concise"
},
"dimensionCompression": "lz4",
"metricCompression": "lz4",
"longEncoding": "longs"
},
"indexSpecForIntermediatePersists": {
"bitmap": {
"type": "concise"
},
"dimensionCompression": "lz4",
"metricCompression": "lz4",
"longEncoding": "longs"
},
"buildV9Directly": true,
"reportParseExceptions": false,
"handoffConditionTimeout": 0,
"resetOffsetAutomatically": false,
"segmentWriteOutMediumFactory": null,
"workerThreads": null,
"chatThreads": null,
"chatRetries": 8,
"httpTimeout": "PT10S",
"shutdownTimeout": "PT80S",
"offsetFetchPeriod": "PT30S",
"intermediateHandoffPeriod": "P2147483647D",
"logParseExceptions": false,
"maxParseExceptions": 2147483647,
"maxSavedParseExceptions": 0,
"skipSequenceNumberAvailabilityCheck": false,
"repartitionTransitionDuration": "PT120S"
},
"type": "kafka"
}

Related

How to add json data in ConfigMap creation in ArgoCD

I'm trying to create a ConfigMap with ArgoCD.
I've created a volumes.yaml file as such
---
apiVersion: v1
kind: ConfigMap
metadata:
name: persistent-volumes-argo
labels:
grafana_dashboard: "1"
project: "foo"
data:
kubernetes.json: |
{{ .Files.Get "dashboards/persistent-volumes.json" | indent 4 }}
But ArgoCD doesn't seem to be able to read the data, the way a standard Helm deployment would.
I've tried adding the data directly into the ConfigMap as such
(Data omitted for brevity)
---
apiVersion: v1
kind: ConfigMap
metadata:
name: persistent-volumes-argo
labels:
grafana_dashboard: "1"
project: "foo"
data:
kubernetes.json: |
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"limit": 100,
"name": "Annotations & Alerts",
"showIn": 0,
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": 13646,
"graphTooltip": 0,
"iteration": 1659421503107,
"links": [],
"panels": [
{
"collapsed": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 26,
"panels": [],
"title": "Alerts",
"type": "row"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"noValue": "--",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "semi-dark-red",
"value": null
},
{
"color": "light-green",
"value": -0.0001
},
{
"color": "semi-dark-red",
"value": 0.0001
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 8,
"x": 0,
"y": 1
},
"id": 21,
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.0.3",
"targets": [
{
"expr": "count (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"} ) and (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"} )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes{namespace=~\"${k8s_namespace}\"} )) >= (${warning_threshold} / 100)) or vector (0)",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "PVCs Above Warning Threshold",
"type": "stat"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 0,
"mappings": [],
"noValue": "--",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "semi-dark-red",
"value": null
},
{
"color": "light-green",
"value": -0.0001
},
{
"color": "semi-dark-red",
"value": 0.0001
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 8,
"x": 8,
"y": 1
},
"id": 24,
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.0.3",
"targets": [
{
"expr": "count((kube_persistentvolumeclaim_status_phase{namespace=~\"${k8s_namespace}\",phase=\"Pending\"}==1)) or vector(0)",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "PVCs in Pending State",
"transformations": [
{
"id": "organize",
"options": {}
}
],
"type": "stat"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 0,
"mappings": [],
"noValue": "--",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "semi-dark-red",
"value": null
},
{
"color": "light-green",
"value": -0.0001
},
{
"color": "semi-dark-red",
"value": 0.0001
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 8,
"x": 16,
"y": 1
},
"id": 23,
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.0.3",
"targets": [
{
"expr": "count((kube_persistentvolumeclaim_status_phase{namespace=~\"${k8s_namespace}\",phase=\"Lost\"}==1)) or vector(0)",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "PVCs in Lost State",
"transformations": [
{
"id": "organize",
"options": {}
}
],
"type": "stat"
},
{
"collapsed": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 17,
"panels": [],
"title": "Usage statistics",
"type": "row"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": null,
"displayMode": "auto",
"filterable": false
},
"mappings": [],
"noValue": "--",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-green",
"value": null
}
]
},
"unit": "none"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Used (%)"
},
"properties": [
{
"id": "custom.displayMode",
"value": "gradient-gauge"
},
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "light-green",
"value": null
},
{
"color": "semi-dark-yellow",
"value": 70
},
{
"color": "dark-red",
"value": 80
}
]
}
},
{
"id": "decimals",
"value": 1
}
]
},
{
"matcher": {
"id": "byName",
"options": "Status"
},
"properties": [
{
"id": "custom.displayMode",
"value": "color-background"
},
{
"id": "mappings",
"value": [
{
"options": {
"0": {
"text": "Bound"
},
"1": {
"text": "Pending"
},
"2": {
"text": "Lost"
}
},
"type": "value"
}
]
},
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "light-green",
"value": null
},
{
"color": "light-green",
"value": 0
},
{
"color": "semi-dark-orange",
"value": 1
},
{
"color": "semi-dark-red",
"value": 2
}
]
}
},
{
"id": "noValue",
"value": "--"
},
{
"id": "custom.align",
"value": "center"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Namespace"
},
"properties": [
{
"id": "custom.width",
"value": 120
}
]
},
{
"matcher": {
"id": "byName",
"options": "Status"
},
"properties": [
{
"id": "custom.width",
"value": 80
}
]
},
{
"matcher": {
"id": "byName",
"options": "Capacity (GiB)"
},
"properties": [
{
"id": "custom.width",
"value": 120
}
]
},
{
"matcher": {
"id": "byName",
"options": "Used (GiB)"
},
"properties": [
{
"id": "custom.width",
"value": 120
}
]
},
{
"matcher": {
"id": "byName",
"options": "Available (GiB)"
},
"properties": [
{
"id": "custom.width",
"value": 120
}
]
},
{
"matcher": {
"id": "byName",
"options": "StorageClass"
},
"properties": [
{
"id": "custom.width",
"value": 150
}
]
},
{
"matcher": {
"id": "byName",
"options": "PersistentVolumeClaim"
},
"properties": [
{
"id": "custom.width",
"value": 370
}
]
}
]
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 6
},
"id": 29,
"interval": "",
"options": {
"frameIndex": 2,
"showHeader": true,
"sortBy": [
{
"desc": false,
"displayName": "PersistentVolumeClaim"
}
]
},
"pluginVersion": "8.0.3",
"targets": [
{
"expr": " sum by (persistentvolumeclaim,namespace,storageclass,volumename) (kube_persistentvolumeclaim_info{namespace=~\"${k8s_namespace}\"})",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
},
{
"expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"${k8s_namespace}\"}/1024/1024/1024)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "B"
},
{
"expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"}/1024/1024/1024)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "C"
},
{
"expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_available_bytes{namespace=~\"${k8s_namespace}\"}/1024/1024/1024)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "D"
},
{
"expr": "sum(kube_persistentvolumeclaim_status_phase{namespace=~\"${k8s_namespace}\",phase=~\"(Pending|Lost)\"}) by (persistentvolumeclaim) + sum(kube_persistentvolumeclaim_status_phase{namespace=~\"${k8s_namespace}\",phase=~\"(Lost)\"}) by (persistentvolumeclaim)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "E"
},
{
"expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"}/kubelet_volume_stats_capacity_bytes{namespace=~\"${k8s_namespace}\"} * 100)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "F"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Persistent Volume Claim",
"transformations": [
{
"id": "seriesToColumns",
"options": {
"byField": "persistentvolumeclaim"
}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Time 1": true,
"Time 2": true,
"Time 3": true,
"Time 4": true,
"Time 5": true,
"Time 6": true,
"Value #A": true
},
"indexByName": {},
"renameByName": {
"Time 1": "",
"Time 2": "",
"Time 3": "",
"Time 4": "",
"Time 5": "",
"Time 6": "",
"Value #A": "",
"Value #B": "Capacity (GiB)",
"Value #C": "Used (GiB)",
"Value #D": "Available (GiB)",
"Value #E": "Status",
"Value #F": "Used (%)",
"namespace": "Namespace",
"persistentvolumeclaim": "PersistentVolumeClaim",
"storageclass": "StorageClass",
"volumename": "PhysicalVolume"
}
}
}
],
"type": "table"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {
"align": null,
"displayMode": "auto",
"filterable": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 18
},
"id": 7,
"options": {
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "Status"
}
]
},
"pluginVersion": "8.0.3",
"targets": [
{
"expr": "kube_storageclass_info",
"format": "table",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Storage Class",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Value": true,
"__name__": true,
"app_kubernetes_io_instance": true,
"app_kubernetes_io_name": true,
"instance": true,
"job": true,
"kubernetes_namespace": true,
"kubernetes_pod_name": true,
"pod_template_hash": true
},
"indexByName": {
"Time": 1,
"Value": 13,
"__name__": 2,
"app_kubernetes_io_instance": 3,
"app_kubernetes_io_name": 4,
"instance": 5,
"job": 6,
"kubernetes_namespace": 7,
"kubernetes_pod_name": 8,
"pod_template_hash": 9,
"provisioner": 10,
"reclaimPolicy": 11,
"storageclass": 0,
"volumeBindingMode": 12
},
"renameByName": {
"provisioner": "Provisioner",
"reclaimPolicy": "ReclaimPolicy",
"storageclass": "StorageClass",
"volumeBindingMode": "VolumeBindingMode"
}
}
},
{
"id": "groupBy",
"options": {
"fields": {
"Provisioner": {
"aggregations": [],
"operation": "groupby"
},
"ReclaimPolicy": {
"aggregations": [],
"operation": "groupby"
},
"StorageClass": {
"aggregations": [],
"operation": "groupby"
},
"VolumeBindingMode": {
"aggregations": [],
"operation": "groupby"
}
}
}
}
],
"type": "table"
},
{
"collapsed": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 23
},
"id": 15,
"panels": [],
"title": "Graphical usage data ",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 24
},
"hiddenSeries": false,
"id": 9,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": true,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.0.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"}))",
"interval": "",
"legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "All Running PVCs Used Bytes",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "Date & time",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": true,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 36
},
"id": 19,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 41
},
"hiddenSeries": false,
"id": 11,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.2.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"}[1h])",
"instant": false,
"interval": "",
"legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Hourly Volume Usage Rate",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "binBps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "Date & time",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 48
},
"hiddenSeries": false,
"id": 12,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.2.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(kubelet_volume_stats_used_bytes{namespace=~\"${k8s_namespace}\"}[1d])",
"interval": "",
"legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Daily Volume Usage Rate",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "binBps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "Date & time",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 55
},
"hiddenSeries": false,
"id": 13,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
}
}
But this errors with rpc error: code = FailedPrecondition desc = Failed to unmarshal "volumes.yaml": <nil>
Is there a way to pass in json data when creating a ConfigMap with ArgoCD, either as a template or by dumping the data in the file?
To create configmap with argocd and helm
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "helm-chart.fullname" . }}-configmap
data:
config.json : |
{{ toJson .Values.configmap | indent 4 }}
and the value file should be like this, which is our JSON but converted to YAML
configmap:
json:
- rigid
- better for data interchange
yaml:
- slim and flexible
- better for configuration
object:
key: value
array:
- null_value:
- boolean: true
- integer: 1
- alias: &example aliases are like variables
- alias: *example
paragraph: >
Blank lines denote
paragraph breaks
content: |-
Or we
can auto
convert line breaks
to save space
alias: &foo
bar: baz
alias_reuse: *foo
json2yaml
A complete demo app can be found here
A very quick way to create app with the above configmap and demo app
argocd app create demo-app --repo https://github.com/Adiii717/argocd-demo-app.git --path helm-chart --dest-namespace default --dest-server https://kubernetes.default.svc --helm-set replicaCount=2
Make sure you also set
export ARGOCD_AUTH_TOKEN="tokeh
export ARGOCD_OPTS="--grpc-web"
export ARGOCD_SERVER="argocd.example.com"
added section for the configmap

How to group by single field and return more values together

I'm starting to use apache druid but having some difficult to run native queries (and some SQL too).
1- Is it possible to groupBy a single column while also returning more channels?
2- How could I groupBy a single column, while returning different grouped itens on same query/row ?
Query I'm trying to use:
{
"queryType": "groupBy",
"dataSource": "my-data-source",
"granularity": "all",
"intervals": ["2022-06-27T03:00:00.000Z/2022-06-28T03:00:00.000Z"],
"context:": { "timeout: 30000 },
"dimensions": ["userId"],
"filter": {
"type": "and",
"fields": [
{
"type": "or",
"fields": [{...}]
}
]
},
"aggregations": [
{
"type": "count",
"name": "count"
}
]
}
Tried to add a filtered type inside aggregations:[] but 0 changes happened.
"aggregations": [
{
"type: "count",
"name": "count"
},
{
"type": "filtered",
"filter": {
"type": "selector",
"dimension": "block_id",
"value": "block1"
},
"aggregator": {
"type": "count",
"name": "block1",
"fieldName": "block_id"
}
}
]
Grouping Aggregator also didn't work.
"aggregations": [
{
"type": "count",
"name": "count"
},
{
"type": "grouping",
"name": "groupedData",
"groupings": ["block_id"]
}
],
Below is the image illustrating the results I'm trying to achieve.
Not sure yet how to get the results in the format you want, but as a start, something like this might be a step:
{
"queryType": "groupBy",
"dataSource": {
"type": "table",
"name": "dataTest"
},
"intervals": {
"type": "intervals",
"intervals": [
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"
]
},
"filter": null,
"granularity": {
"type": "all"
},
"dimensions": [
{
"type": "default",
"dimension": "d2_ts2",
"outputType": "STRING"
},
{
"type": "default",
"dimension": "d3_email",
"outputType": "STRING"
}
],
"aggregations": [
{
"type": "count",
"name": "myCount",
}
],
"descending": false
}
I'm curious, what is the use case?
Using a SQL query you can do it this way:
SELECT UserID,
sum(1) FILTER (WHERE BlockId = 'block1') as Block1,
sum(1) FILTER (WHERE BlockId = 'block2') as Block2,
sum(1) FILTER (WHERE BlockId = 'block3') as Block3
FROM inline_data
GROUP BY 1
The Native Query for this (from the explain) is:
{
"queryType": "topN",
"dataSource": {
"type": "table",
"name": "inline_data"
},
"virtualColumns": [
{
"type": "expression",
"name": "v0",
"expression": "1",
"outputType": "LONG"
}
],
"dimension": {
"type": "default",
"dimension": "UserID",
"outputName": "d0",
"outputType": "STRING"
},
"metric": {
"type": "dimension",
"previousStop": null,
"ordering": {
"type": "lexicographic"
}
},
"threshold": 101,
"intervals": {
"type": "intervals",
"intervals": [
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"
]
},
"filter": null,
"granularity": {
"type": "all"
},
"aggregations": [
{
"type": "filtered",
"aggregator": {
"type": "longSum",
"name": "a0",
"fieldName": "v0",
"expression": null
},
"filter": {
"type": "selector",
"dimension": "BlockId",
"value": "block1",
"extractionFn": null
},
"name": "a0"
},
{
"type": "filtered",
"aggregator": {
"type": "longSum",
"name": "a1",
"fieldName": "v0",
"expression": null
},
"filter": {
"type": "selector",
"dimension": "BlockId",
"value": "block2",
"extractionFn": null
},
"name": "a1"
},
{
"type": "filtered",
"aggregator": {
"type": "longSum",
"name": "a2",
"fieldName": "v0",
"expression": null
},
"filter": {
"type": "selector",
"dimension": "BlockId",
"value": "block3",
"extractionFn": null
},
"name": "a2"
}
],
"postAggregations": [],
"context": {
"populateCache": false,
"sqlOuterLimit": 101,
"sqlQueryId": "bb92e899-c127-49b0-be1b-d4b38909d166",
"useApproximateCountDistinct": false,
"useApproximateTopN": false,
"useCache": false,
"useNativeQueryExplain": true
},
"descending": false
}

org.apache.kafka.connect.transforms.ReplaceField does not work

The documentation I used: https://docs.confluent.io/platform/current/connect/transforms/replacefield.html
I use this connector to rename PersonId column to Id by using the org.apache.kafka.connect.transforms.ReplaceField and setting renames to PersonId:Id
{
"name": "SQL_Connector",
"config": {
"connector.class": "io.debezium.connector.sqlserver.SqlServerConnector",
"tasks.max": "1",
"database.hostname": "hostname",
"database.port": "1433",
"database.user": "user",
"database.password": "password",
"database.dbname": "sqlserver",
"database.server.name": "server",
"database.history.kafka.bootstrap.servers": "kafka:9092",
"database.history.kafka.topic": "dbhistory.test",
"transforms": "RenameField,addStaticField",
"transforms.RenameField.type": "org.apache.kafka.connect.transforms.ReplaceField$Value",
"transforms.RenameField.renames": "PersonId:Id",
"transforms.addStaticField.type":"org.apache.kafka.connect.transforms.InsertField$Value",
"transforms.addStaticField.static.field":"table",
"transforms.addStaticField.static.value":"changedtablename",
}
}
But when I get the value in the topic the field PersonId is not changed:
{
"schema": {
"type": "struct",
"fields": [
{
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "PersonId"
}
],
"optional": true,
"name": "test.Value",
"field": "before"
},
{
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "PersonId"
}
],
"optional": true,
"name": "test.Value",
"field": "after"
},
{
"type": "struct",
"fields": [
{
"type": "string",
"optional": false,
"field": "version"
},
{
"type": "string",
"optional": false,
"field": "connector"
},
{
"type": "string",
"optional": false,
"field": "name"
},
{
"type": "int64",
"optional": false,
"field": "ts_ms"
},
{
"type": "string",
"optional": true,
"name": "io.debezium.data.Enum",
"version": 1,
"parameters": {
"allowed": "true,last,false"
},
"default": "false",
"field": "snapshot"
},
{
"type": "string",
"optional": false,
"field": "db"
},
{
"type": "string",
"optional": false,
"field": "schema"
},
{
"type": "string",
"optional": false,
"field": "table"
},
{
"type": "string",
"optional": true,
"field": "change_lsn"
},
{
"type": "string",
"optional": true,
"field": "commit_lsn"
},
{
"type": "int64",
"optional": true,
"field": "event_serial_no"
}
],
"optional": false,
"name": "io.debezium.connector.sqlserver.Source",
"field": "source"
},
{
"type": "string",
"optional": false,
"field": "op"
},
{
"type": "int64",
"optional": true,
"field": "ts_ms"
},
{
"type": "string",
"optional": true,
"field": "table"
}
],
"optional": false,
"name": "test.Envelope"
},
"payload": {
"before": null,
"after": {
"PersonId": 1,
},
"source": {
"version": "1.0.3.Final",
"connector": "sqlserver",
"name": "test",
"ts_ms": 1627628793596,
"snapshot": "true",
"db": "test",
"schema": "dbo",
"table": "TestTable",
"change_lsn": null,
"commit_lsn": "00023472:00000100:0001",
"event_serial_no": null
},
"op": "r",
"ts_ms": 1627628793596,
"table": "changedtablename"
}
}
How do I change the Field?
You can only replace fields that are at the top-level of the Kafka Record, as the example in the doc shows.
That being said, you will need to extract the after field first

Need primary key information in Debezium connector for postgres insert events

I am using Debezium connector for postgres with Kafka connect.
For an insert row event written to Kafka by the connector, I need information about which columns are primary keys and which are not. Is there a way to achieve this ?
Pasting a sample insert event generated in Kafka:
"schema": {
"type": "struct",
"fields": [
{
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "id"
},
{
"type": "int32",
"optional": false,
"field": "bucket_type"
}
],
"optional": true,
"name": "postgresconfigdb.config.alert_configs.Value",
"field": "before"
},
{
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "id"
},
{
"type": "int32",
"optional": false,
"field": "bucket_type"
}
],
"optional": true,
"name": "postgresconfigdb.config.alert_configs.Value",
"field": "after"
},
{
"type": "struct",
"fields": [
{
"type": "string",
"optional": false,
"field": "version"
},
{
"type": "string",
"optional": false,
"field": "connector"
},
{
"type": "string",
"optional": false,
"field": "name"
},
{
"type": "int64",
"optional": false,
"field": "ts_ms"
},
{
"type": "string",
"optional": true,
"name": "io.debezium.data.Enum",
"version": 1,
"parameters": {
"allowed": "true,last,false"
},
"default": "false",
"field": "snapshot"
},
{
"type": "string",
"optional": false,
"field": "db"
},
{
"type": "string",
"optional": false,
"field": "schema"
},
{
"type": "string",
"optional": false,
"field": "table"
},
{
"type": "int64",
"optional": true,
"field": "txId"
},
{
"type": "int64",
"optional": true,
"field": "lsn"
},
{
"type": "int64",
"optional": true,
"field": "xmin"
}
],
"optional": false,
"name": "io.debezium.connector.postgresql.Source",
"field": "source"
},
{
"type": "string",
"optional": false,
"field": "op"
},
{
"type": "int64",
"optional": true,
"field": "ts_ms"
},
{
"type": "struct",
"fields": [
{
"type": "string",
"optional": false,
"field": "id"
},
{
"type": "int64",
"optional": false,
"field": "total_order"
},
{
"type": "int64",
"optional": false,
"field": "data_collection_order"
}
],
"optional": true,
"field": "transaction"
}
],
"optional": false,
"name": "postgresconfigdb.config.alert_configs.Envelope"
},
"payload": {
"before": null,
"after": {
"id": 1100,
"bucket_type": 10
},
"source": {
"version": "1.2.0.Final",
"connector": "postgresql",
"name": "postgresconfigdb",
"ts_ms": 1599830887858,
"snapshot": "true",
"db": "configdb",
"schema": "config",
"table": "alert_configs",
"txId": 2139888,
"lsn": 379356048,
"xmin": null
},
"op": "r",
"ts_ms": 1599830887859,
"transaction": null
}
}
Here the columns in the table are 'id' and 'bucket_type', the values of which are reported in the json-path payload->after.
There is information about columns which are not null in the column specific 'optional' boolean field, however no information about which columns are primary keys. (id in this case)
you find information about what fields are PK columns in Kafka key.

JOLT Flatten Unnamed nested array and Split function

Working on transformation and created spec for the input. The output is nested array but I am expecting flattened array.
2. Need to split the string and get the first element. split function doesn't seems to be working.
Please find the input, spec , output and expected output below.
Input
[
{
"tables": [
{
"columns": [
{
"size": 20,
"nullable": false,
"databaseSpecificType": "varchar",
"generated": false,
"dataType": "VARCHAR",
"name": "firstname",
"width": "(20)",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
},
{
"size": 20,
"nullable": false,
"databaseSpecificType": "varchar",
"generated": false,
"dataType": "VARCHAR",
"name": "lastname",
"width": "(20)",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
}
],
"name": "authors",
"fullName": "books.authors",
"type": "table",
"triggers": [],
"tableConstraints": [],
"remarks": "Contact details for book authors",
"primaryKey": {
"columns": {
"sortSequence": "ascending",
"name": "id"
},
"unique": true,
"name": "pk_authors",
"remarks": ""
}
},
{
"columns": [
{
"size": 10,
"nullable": true,
"databaseSpecificType": "int4",
"generated": false,
"dataType": "INTEGER",
"name": "id",
"width": "",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
}
],
"name": "authorslist",
"fullName": "books.authorslist",
"type": "view",
"triggers": [],
"tableConstraints": [],
"remarks": "",
"primaryKey": {}
}
],
"schemaCrawlerHeaderInfo": {
"crawlTimestamp": "2018-05-23 10:21:55",
"title": ""
}
}
]
Spec:
[{
"operation": "shift",
"spec": {
"*": {
"tables": {
"*": {
"columns": {
"*": {
"#(2,name)": "[&3].[&1].TABLE.tableName",
"#(2,fullName)": ["[&3].[&1].TABLE.fullName", "[&3].[&1].DB.fullName"],
"#(2,remarks)": "[&3].[&1].TABLE.tableDesc",
"name": "[&3].[&1].COLUMN.name",
"dataType": "[&3].[&1].COLUMN.dataType",
"size": "[&3].[&1].COLUMN.size",
"nullable": "[&3].[&1].COLUMN.nullable",
"databaseSpecificType": "[&3].[&1].COLUMN.databaseSpecificType",
"width": "[&3].[&1].COLUMN.width",
"decimalDigits": "[&3].[&1].COLUMN.decimalDigits",
"remarks": "[&3].[&1].COLUMN.remarks",
"autoIncremented": "[&3].[&1].COLUMN.autoIncremented"
}
}
}
}
}
}
}]
Output
[
[
{
"TABLE": {
"tableName": "authors",
"fullName": "books.authors",
"tableDesc": "Contact details for book authors"
},
"DB": {
"fullName": "books.authors"
},
"COLUMN": {
"name": "firstname",
"dataType": "VARCHAR",
"size": 20,
"nullable": false,
"databaseSpecificType": "varchar",
"width": "(20)",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
}
},
{
"TABLE": {
"tableName": "authors",
"fullName": "books.authors",
"tableDesc": "Contact details for book authors"
},
"DB": {
"fullName": "books.authors"
},
"COLUMN": {
"name": "lastname",
"dataType": "VARCHAR",
"size": 20,
"nullable": false,
"databaseSpecificType": "varchar",
"width": "(20)",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
}
}
],
[
{
"TABLE": {
"tableName": "authorslist",
"fullName": "books.authorslist",
"tableDesc": ""
},
"DB": {
"fullName": "books.authorslist"
},
"COLUMN": {
"name": "id",
"dataType": "INTEGER",
"size": 10,
"nullable": true,
"databaseSpecificType": "int4",
"width": "",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
}
}
]
]Iamexpectingtheflattenedoutputlikebelowinsinglearrayrathernestedarray.[
{
"TABLE": {
"tableName": "authors",
"fullName": "books.authors",
"tableDesc": "Contact details for book authors"
},
"DB": {
"fullName": "books.authors"
},
"COLUMN": {
"name": "firstname",
"dataType": "VARCHAR",
"size": 20,
"nullable": false,
"databaseSpecificType": "varchar",
"width": "(20)",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
}
},
{
"TABLE": {
"tableName": "authors",
"fullName": "books.authors",
"tableDesc": "Contact details for book authors"
},
"DB": {
"fullName": "books.authors"
},
"COLUMN": {
"name": "lastname",
"dataType": "VARCHAR",
"size": 20,
"nullable": false,
"databaseSpecificType": "varchar",
"width": "(20)",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
}
},
{
"TABLE": {
"tableName": "authorslist",
"fullName": "books.authorslist",
"tableDesc": ""
},
"DB": {
"fullName": "books.authorslist"
},
"COLUMN": {
"name": "id",
"dataType": "INTEGER",
"size": 10,
"nullable": true,
"databaseSpecificType": "int4",
"width": "",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
}
}
]
I am expecting the flattened output like below in single array rather nested array.
[
{
"TABLE": {
"tableName": "authors",
"fullName": "books.authors",
"tableDesc": "Contact details for book authors"
},
"DB": {
"fullName": "books.authors"
},
"COLUMN": {
"name": "firstname",
"dataType": "VARCHAR",
"size": 20,
"nullable": false,
"databaseSpecificType": "varchar",
"width": "(20)",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
}
},
{
"TABLE": {
"tableName": "authors",
"fullName": "books.authors",
"tableDesc": "Contact details for book authors"
},
"DB": {
"fullName": "books.authors"
},
"COLUMN": {
"name": "lastname",
"dataType": "VARCHAR",
"size": 20,
"nullable": false,
"databaseSpecificType": "varchar",
"width": "(20)",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
}
},
{
"TABLE": {
"tableName": "authorslist",
"fullName": "books.authorslist",
"tableDesc": ""
},
"DB": {
"fullName": "books.authorslist"
},
"COLUMN": {
"name": "id",
"dataType": "INTEGER",
"size": 10,
"nullable": true,
"databaseSpecificType": "int4",
"width": "",
"decimalDigits": 0,
"remarks": "",
"autoIncremented": false
}
}
]
And also tried split and firstelement function to get the dbname but split is not working.
"DB" : {
"fullName" : "books.authors"
}
"DB" : {
"fullName" : "books"
}
Any help on this would be great.
Have to do two shifts.
Spec and explaination
[
{
// Your input data is two nested arrays.
// Tables array, and its nested columns array.
// The number of elements in your output array is
// the total number of column elements in your input doc.
//
// Can't do it in a single shift.
// The sample input tables arrays is length 2
// The sample input columsn arrays are length 2 and 1.
// You want an output array of length 3, there is no "3"
// that can be referenced by the first shift operation.
//
// So first, "build" the final TABLE, COLUMN, DB format, by
// pushing table information down into the columns data, BUT
// keeping the same nested table and columns array structure.
"operation": "shift",
"spec": {
"0": {
"tables": {
"*": { // tables index
"columns": {
"*": { // columns index
// grabbing info off the "parent" table entry
"#(2,name)": "tables[&3].cols[&1].TABLE.tableName",
"#(2,fullName)": ["tables[&3].cols[&1].TABLE.fullName", "tables[&3].cols[&1].DB.fullName"],
"#(2,remarks)": "tables[&3].cols[&1].TABLE.tableDesc",
//
// handling all the column level data
"*": "tables[&3].cols[&1].COLUMN.&"
}
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"tables": {
"*": {
"cols": {
// now walk the nested tables and cols array
// and accumulate each "fully formatted"
// col entry, into the output top-level array
"*": "[]"
}
}
}
}
}
]