Related
when install kubernetes dashboard using this command:
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml
I check kubernetes service state like this:
kubectl get events --namespace=kube-system
It give me this tip:
LAST SEEN TYPE REASON OBJECT MESSAGE
2m38s Warning ClusterIPOutOfRange service/kube-dns Cluster IP 10.43.0.10 is not within the service CIDR 10.254.0.0/16; please recreate service
2m38s Warning ClusterIPOutOfRange service/metrics-server Cluster IP 10.43.96.112 is not within the service CIDR 10.254.0.0/16; please recreate service
is there any way to change my kube-dns service ip into CIDR range(I search from internet and no one change to ip by hand,should I tweak the kube-dns component config)?This is version info:
[root#iZuf63refzweg1d9dh94t8Z ~]# kubectl version
Client Version: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.2", GitCommit:"f6278300bebbb750328ac16ee6dd3aa7d3549568", GitTreeState:"clean", BuildDate:"2019-08-05T09:23:26Z", GoVersion:"go1.12.5", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.2", GitCommit:"f6278300bebbb750328ac16ee6dd3aa7d3549568", GitTreeState:"clean", BuildDate:"2019-08-05T09:15:22Z", GoVersion:"go1.12.5", Compiler:"gc", Platform:"linux/amd64"}
this is cluster dump:
{
"kind": "NodeList",
"apiVersion": "v1",
"metadata": {
"selfLink": "/api/v1/nodes",
"resourceVersion": "73105"
},
"items": []
}
{
"kind": "EventList",
"apiVersion": "v1",
"metadata": {
"selfLink": "/api/v1/namespaces/kube-system/events",
"resourceVersion": "8044226"
},
"items": [
{
"metadata": {
"name": "kube-dns.15bf7a181c6f8459",
"namespace": "kube-system",
"selfLink": "/api/v1/namespaces/kube-system/events/kube-dns.15bf7a181c6f8459",
"uid": "3adfac28-ca39-4cc0-a516-08220a4be8b5",
"resourceVersion": "8044216",
"creationTimestamp": "2019-08-29T19:03:59Z"
},
"involvedObject": {
"kind": "Service",
"namespace": "kube-system",
"name": "kube-dns",
"uid": "2256b0f5-80f5-11e9-b3ce-00163e086f0c",
"apiVersion": "v1",
"resourceVersion": "379"
},
"reason": "ClusterIPOutOfRange",
"message": "Cluster IP 10.43.0.10 is not within the service CIDR 10.254.0.0/16; please recreate service",
"source": {
"component": "ipallocator-repair-controller"
},
"firstTimestamp": "2019-08-29T19:03:59Z",
"lastTimestamp": "2019-09-05T15:10:58Z",
"count": 3283,
"type": "Warning",
"eventTime": null,
"reportingComponent": "",
"reportingInstance": ""
},
{
"metadata": {
"name": "metrics-server.15bf7a181c8012e4",
"namespace": "kube-system",
"selfLink": "/api/v1/namespaces/kube-system/events/metrics-server.15bf7a181c8012e4",
"uid": "0be9374b-b497-4a9d-86d2-2b75da47b659",
"resourceVersion": "8044217",
"creationTimestamp": "2019-08-29T19:03:59Z"
},
"involvedObject": {
"kind": "Service",
"namespace": "kube-system",
"name": "metrics-server",
"uid": "255955e6-80f5-11e9-b3ce-00163e086f0c",
"apiVersion": "v1",
"resourceVersion": "427"
},
"reason": "ClusterIPOutOfRange",
"message": "Cluster IP 10.43.96.112 is not within the service CIDR 10.254.0.0/16; please recreate service",
"source": {
"component": "ipallocator-repair-controller"
},
"firstTimestamp": "2019-08-29T19:03:59Z",
"lastTimestamp": "2019-09-05T15:10:58Z",
"count": 3283,
"type": "Warning",
"eventTime": null,
"reportingComponent": "",
"reportingInstance": ""
}
]
}
{
"kind": "ReplicationControllerList",
"apiVersion": "v1",
"metadata": {
"selfLink": "/api/v1/namespaces/kube-system/replicationcontrollers",
"resourceVersion": "73105"
},
"items": []
}
{
"kind": "ServiceList",
"apiVersion": "v1",
"metadata": {
"selfLink": "/api/v1/namespaces/kube-system/services",
"resourceVersion": "8044226"
},
"items": [
{
"metadata": {
"name": "kube-dns",
"namespace": "kube-system",
"selfLink": "/api/v1/namespaces/kube-system/services/kube-dns",
"uid": "2256b0f5-80f5-11e9-b3ce-00163e086f0c",
"resourceVersion": "379",
"creationTimestamp": "2019-05-28T03:03:14Z",
"labels": {
"addonmanager.kubernetes.io/mode": "Reconcile",
"k8s-app": "kube-dns",
"kubernetes.io/cluster-service": "true",
"kubernetes.io/name": "KubeDNS"
},
"annotations": {
"kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{},\"labels\":{\"addonmanager.kubernetes.io/mode\":\"Reconcile\",\"k8s-app\":\"kube-dns\",\"kubernetes.io/cluster-service\":\"true\",\"kubernetes.io/name\":\"KubeDNS\"},\"name\":\"kube-dns\",\"namespace\":\"kube-system\"},\"spec\":{\"clusterIP\":\"10.43.0.10\",\"ports\":[{\"name\":\"dns\",\"port\":53,\"protocol\":\"UDP\"},{\"name\":\"dns-tcp\",\"port\":53,\"protocol\":\"TCP\"}],\"selector\":{\"k8s-app\":\"kube-dns\"}}}\n"
}
},
"spec": {
"ports": [
{
"name": "dns",
"protocol": "UDP",
"port": 53,
"targetPort": 53
},
{
"name": "dns-tcp",
"protocol": "TCP",
"port": 53,
"targetPort": 53
}
],
"selector": {
"k8s-app": "kube-dns"
},
"clusterIP": "10.43.0.10",
"type": "ClusterIP",
"sessionAffinity": "None"
},
"status": {
"loadBalancer": {}
}
},
{
"metadata": {
"name": "metrics-server",
"namespace": "kube-system",
"selfLink": "/api/v1/namespaces/kube-system/services/metrics-server",
"uid": "255955e6-80f5-11e9-b3ce-00163e086f0c",
"resourceVersion": "427",
"creationTimestamp": "2019-05-28T03:03:19Z",
"labels": {
"kubernetes.io/name": "Metrics-server"
},
"annotations": {
"kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{},\"labels\":{\"kubernetes.io/name\":\"Metrics-server\"},\"name\":\"metrics-server\",\"namespace\":\"kube-system\"},\"spec\":{\"ports\":[{\"port\":443,\"protocol\":\"TCP\",\"targetPort\":443}],\"selector\":{\"k8s-app\":\"metrics-server\"}}}\n"
}
},
"spec": {
"ports": [
{
"protocol": "TCP",
"port": 443,
"targetPort": 443
}
],
"selector": {
"k8s-app": "metrics-server"
},
"clusterIP": "10.43.96.112",
"type": "ClusterIP",
"sessionAffinity": "None"
},
"status": {
"loadBalancer": {}
}
}
]
}
{
"kind": "DaemonSetList",
"apiVersion": "apps/v1",
"metadata": {
"selfLink": "/apis/apps/v1/namespaces/kube-system/daemonsets",
"resourceVersion": "73105"
},
"items": []
}
{
"kind": "DeploymentList",
"apiVersion": "apps/v1",
"metadata": {
"selfLink": "/apis/apps/v1/namespaces/kube-system/deployments",
"resourceVersion": "73105"
},
"items": []
}
{
"kind": "ReplicaSetList",
"apiVersion": "apps/v1",
"metadata": {
"selfLink": "/apis/apps/v1/namespaces/kube-system/replicasets",
"resourceVersion": "73105"
},
"items": [
{
"metadata": {
"name": "kubernetes-dashboard-7d75c474bb",
"namespace": "kube-system",
"selfLink": "/apis/apps/v1/namespaces/kube-system/replicasets/kubernetes-dashboard-7d75c474bb",
"uid": "1b426257-5d74-4f50-b368-45f65d926fdf",
"resourceVersion": "57629",
"generation": 1,
"creationTimestamp": "2019-08-29T15:11:10Z",
"labels": {
"k8s-app": "kubernetes-dashboard",
"pod-template-hash": "7d75c474bb"
},
"annotations": {
"deployment.kubernetes.io/desired-replicas": "1",
"deployment.kubernetes.io/max-replicas": "2",
"deployment.kubernetes.io/revision": "1"
},
"ownerReferences": [
{
"apiVersion": "apps/v1",
"kind": "Deployment",
"name": "kubernetes-dashboard",
"uid": "c5b2daaa-d306-43b1-ab0a-9745beb865de",
"controller": true,
"blockOwnerDeletion": true
}
]
},
"spec": {
"replicas": 1,
"selector": {
"matchLabels": {
"k8s-app": "kubernetes-dashboard",
"pod-template-hash": "7d75c474bb"
}
},
"template": {
"metadata": {
"creationTimestamp": null,
"labels": {
"k8s-app": "kubernetes-dashboard",
"pod-template-hash": "7d75c474bb"
}
},
"spec": {
"volumes": [
{
"name": "kubernetes-dashboard-certs",
"secret": {
"secretName": "kubernetes-dashboard-certs",
"defaultMode": 420
}
},
{
"name": "tmp-volume",
"emptyDir": {}
}
],
"containers": [
{
"name": "kubernetes-dashboard",
"image": "k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1",
"args": [
"--auto-generate-certificates"
],
"ports": [
{
"containerPort": 8443,
"protocol": "TCP"
}
],
"resources": {},
"volumeMounts": [
{
"name": "kubernetes-dashboard-certs",
"mountPath": "/certs"
},
{
"name": "tmp-volume",
"mountPath": "/tmp"
}
],
"livenessProbe": {
"httpGet": {
"path": "/",
"port": 8443,
"scheme": "HTTPS"
},
"initialDelaySeconds": 30,
"timeoutSeconds": 30,
"periodSeconds": 10,
"successThreshold": 1,
"failureThreshold": 3
},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent"
}
],
"restartPolicy": "Always",
"terminationGracePeriodSeconds": 30,
"dnsPolicy": "ClusterFirst",
"serviceAccountName": "kubernetes-dashboard",
"serviceAccount": "kubernetes-dashboard",
"securityContext": {},
"schedulerName": "default-scheduler",
"tolerations": [
{
"key": "node-role.kubernetes.io/master",
"effect": "NoSchedule"
}
]
}
}
},
"status": {
"replicas": 1,
"fullyLabeledReplicas": 1,
"observedGeneration": 1
}
}
]
}
{
"kind": "PodList",
"apiVersion": "v1",
"metadata": {
"selfLink": "/api/v1/namespaces/kube-system/pods",
"resourceVersion": "73105"
},
"items": [
{
"metadata": {
"name": "kubernetes-dashboard-7d75c474bb-b2lwd",
"generateName": "kubernetes-dashboard-7d75c474bb-",
"namespace": "kube-system",
"selfLink": "/api/v1/namespaces/kube-system/pods/kubernetes-dashboard-7d75c474bb-b2lwd",
"uid": "d4013a3c-7688-4a00-8acf-e5e49c10c772",
"resourceVersion": "57626",
"creationTimestamp": "2019-08-29T15:11:10Z",
"labels": {
"k8s-app": "kubernetes-dashboard",
"pod-template-hash": "7d75c474bb"
},
"ownerReferences": [
{
"apiVersion": "apps/v1",
"kind": "ReplicaSet",
"name": "kubernetes-dashboard-7d75c474bb",
"uid": "1b426257-5d74-4f50-b368-45f65d926fdf",
"controller": true,
"blockOwnerDeletion": true
}
]
},
"spec": {
"volumes": [
{
"name": "kubernetes-dashboard-certs",
"secret": {
"secretName": "kubernetes-dashboard-certs",
"defaultMode": 420
}
},
{
"name": "tmp-volume",
"emptyDir": {}
},
{
"name": "kubernetes-dashboard-token-7k8wl",
"secret": {
"secretName": "kubernetes-dashboard-token-7k8wl",
"defaultMode": 420
}
}
],
"containers": [
{
"name": "kubernetes-dashboard",
"image": "k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1",
"args": [
"--auto-generate-certificates"
],
"ports": [
{
"containerPort": 8443,
"protocol": "TCP"
}
],
"resources": {},
"volumeMounts": [
{
"name": "kubernetes-dashboard-certs",
"mountPath": "/certs"
},
{
"name": "tmp-volume",
"mountPath": "/tmp"
},
{
"name": "kubernetes-dashboard-token-7k8wl",
"readOnly": true,
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
}
],
"livenessProbe": {
"httpGet": {
"path": "/",
"port": 8443,
"scheme": "HTTPS"
},
"initialDelaySeconds": 30,
"timeoutSeconds": 30,
"periodSeconds": 10,
"successThreshold": 1,
"failureThreshold": 3
},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent"
}
],
"restartPolicy": "Always",
"terminationGracePeriodSeconds": 30,
"dnsPolicy": "ClusterFirst",
"serviceAccountName": "kubernetes-dashboard",
"serviceAccount": "kubernetes-dashboard",
"securityContext": {},
"schedulerName": "default-scheduler",
"tolerations": [
{
"key": "node-role.kubernetes.io/master",
"effect": "NoSchedule"
}
],
"enableServiceLinks": true
},
"status": {
"phase": "Pending",
"conditions": [
{
"type": "PodScheduled",
"status": "False",
"lastProbeTime": null,
"lastTransitionTime": "2019-08-29T15:11:10Z",
"reason": "Unschedulable",
"message": "no nodes available to schedule pods"
}
],
"qosClass": "BestEffort"
}
}
]
}
==== START logs for container kubernetes-dashboard of pod kube-system/kubernetes-dashboard-7d75c474bb-b2lwd ====
==== END logs for container kubernetes-dashboard of pod kube-system/kubernetes-dashboard-7d75c474bb-b2lwd ====
{
"kind": "EventList",
"apiVersion": "v1",
"metadata": {
"selfLink": "/api/v1/namespaces/default/events",
"resourceVersion": "8044226"
},
"items": [
{
"metadata": {
"name": "kubernetes.15bf7a181c3cf6a8",
"namespace": "default",
"selfLink": "/api/v1/namespaces/default/events/kubernetes.15bf7a181c3cf6a8",
"uid": "4502f2e2-efd1-4707-b402-bfb16194cd52",
"resourceVersion": "8044213",
"creationTimestamp": "2019-08-29T19:03:59Z"
},
"involvedObject": {
"kind": "Service",
"namespace": "default",
"name": "kubernetes",
"uid": "089121d4-80f5-11e9-b3ce-00163e086f0c",
"apiVersion": "v1",
"resourceVersion": "6"
},
"reason": "ClusterIPOutOfRange",
"message": "Cluster IP 10.43.0.1 is not within the service CIDR 10.254.0.0/16; please recreate service",
"source": {
"component": "ipallocator-repair-controller"
},
"firstTimestamp": "2019-08-29T19:03:59Z",
"lastTimestamp": "2019-09-05T15:10:58Z",
"count": 3283,
"type": "Warning",
"eventTime": null,
"reportingComponent": "",
"reportingInstance": ""
}
]
}
{
"kind": "ReplicationControllerList",
"apiVersion": "v1",
"metadata": {
"selfLink": "/api/v1/namespaces/default/replicationcontrollers",
"resourceVersion": "73105"
},
"items": []
}
{
"kind": "ServiceList",
"apiVersion": "v1",
"metadata": {
"selfLink": "/api/v1/namespaces/default/services",
"resourceVersion": "8044226"
},
"items": [
{
"metadata": {
"name": "kubernetes",
"namespace": "default",
"selfLink": "/api/v1/namespaces/default/services/kubernetes",
"uid": "089121d4-80f5-11e9-b3ce-00163e086f0c",
"resourceVersion": "6",
"creationTimestamp": "2019-05-28T03:02:31Z",
"labels": {
"component": "apiserver",
"provider": "kubernetes"
}
},
"spec": {
"ports": [
{
"name": "https",
"protocol": "TCP",
"port": 443,
"targetPort": 6443
}
],
"clusterIP": "10.43.0.1",
"type": "ClusterIP",
"sessionAffinity": "None"
},
"status": {
"loadBalancer": {}
}
}
]
}
{
"kind": "DaemonSetList",
"apiVersion": "apps/v1",
"metadata": {
"selfLink": "/apis/apps/v1/namespaces/default/daemonsets",
"resourceVersion": "73105"
},
"items": []
}
{
"kind": "DeploymentList",
"apiVersion": "apps/v1",
"metadata": {
"selfLink": "/apis/apps/v1/namespaces/default/deployments",
"resourceVersion": "73105"
},
"items": []
}
{
"kind": "ReplicaSetList",
"apiVersion": "apps/v1",
"metadata": {
"selfLink": "/apis/apps/v1/namespaces/default/replicasets",
"resourceVersion": "73105"
},
"items": []
}
{
"kind": "PodList",
"apiVersion": "v1",
"metadata": {
"selfLink": "/api/v1/namespaces/default/pods",
"resourceVersion": "73105"
},
"items": []
}
Cluster info dumped to standard output
I can obtain my service by running
$ kubectl get service <service-name> --namespace <namespace name>
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service name LoadBalancer ********* ********* port numbers 16h
here is my service running at kubernetes but I can't access it through public IP. below are my service and deployment files added . i am using azre devops to build and release container image to azure container registry . As you see above on service describe i got external ip and cluster ip but when i try this ip in browser or use curl i get no response. `
{
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": "service-name",
"namespace": "namespace-name",
"selfLink": "*******************",
"uid": "*******************",
"resourceVersion": "1686278",
"creationTimestamp": "2019-07-15T14:12:11Z",
"labels": {
"run": "service name"
}
},
"spec": {
"ports": [
{
"protocol": "TCP",
"port": 80,
"targetPort": ****,
"nodePort": ****
}
],
"selector": {
"run": "profile-management-service"
},
"clusterIP": "**********",
"type": "LoadBalancer",
"sessionAffinity": "None",
"externalTrafficPolicy": "Cluster"
},
"status": {
"loadBalancer": {
"ingress": [
{
"ip": "*************"
}
]
}
}
}
{
"kind": "Deployment",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "deployment-name",
"namespace": "namespace-name",
"selfLink": "*************************",
"uid": "****************************",
"resourceVersion": "1686172",
"generation": 1,
"creationTimestamp": "2019-07-15T14:12:04Z",
"labels": {
"run": "deployment-name"
},
"annotations": {
"deployment.kubernetes.io/revision": "1"
}
},
"spec": {
"replicas": 1,
"selector": {
"matchLabels": {
"run": "deployment-name"
}
},
"template": {
"metadata": {
"creationTimestamp": null,
"labels": {
"run": "deployment-name"
}
},
"spec": {
"containers": [
{
"name": "deployment-name",
"image": "dev/containername:50",
"ports": [
{
"containerPort": ****,
"protocol": "TCP"
}
],
"resources": {},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent"
}
],
"restartPolicy": "Always",
"terminationGracePeriodSeconds": 30,
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"schedulerName": "default-scheduler"
}
},
"strategy": {
"type": "RollingUpdate",
"rollingUpdate": {
"maxUnavailable": 1,
"maxSurge": 1
}
},
"revisionHistoryLimit": 2147483647,
"progressDeadlineSeconds": 2147483647
},
"status": {
"observedGeneration": 1,
"replicas": 1,
"updatedReplicas": 1,
"readyReplicas": 1,
"availableReplicas": 1,
"conditions": [
{
"type": "Available",
"status": "True",
"lastUpdateTime": "2019-07-15T14:12:04Z",
"lastTransitionTime": "2019-07-15T14:12:04Z",
"reason": "MinimumReplicasAvailable",
"message": "Deployment has minimum availability."
}
]
}
}
`
Apparently there's a mismatch in label and selector:
Service selector
"selector": {
"run": "profile-management-service"
While deployment label
"labels": {
"run": "deployment-name"
},
Also check targetPort value of the service, it should match containerPort of your deployment
You need to add readinessProbe and livenessProbe on your Deployment and after that check your firewall if all rules are correct.
Here you have some more info about liveness and readiness
I am using kubernetes v1.9.7-gke.6. I am trying to edit the metrics-server deployment yaml and add --metric-resolution flag, when I add the flag and save the change I see on the terminal that the edit was successful. When I edit again the metrics-server deployment file the flag that I added it doesn't exist. Is there any way to edit the metrics server deployment yaml?
Here is the deployment , its the default that created when I create a new kuberentes cluster at google cloud.
{
"apiVersion": "extensions/v1beta1",
"kind": "Deployment",
"metadata": {
"annotations": {
"deployment.kubernetes.io/revision": "12",
"kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"extensions/v1beta1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"addonmanager.kubernetes.io/mode\":\"Reconcile\",\"k8s-app\":\"metrics-server\",\"kubernetes.io/cluster-service\":\"true\",\"version\":\"v0.2.1\"},\"name\":\"metrics-server-v0.2.1\",\"namespace\":\"kube-system\"},\"spec\":{\"selector\":{\"matchLabels\":{\"k8s-app\":\"metrics-server\",\"version\":\"v0.2.1\"}},\"template\":{\"metadata\":{\"annotations\":{\"scheduler.alpha.kubernetes.io/critical-pod\":\"\"},\"labels\":{\"k8s-app\":\"metrics-server\",\"version\":\"v0.2.1\"},\"name\":\"metrics-server\"},\"spec\":{\"containers\":[{\"command\":[\"/metrics-server\",\"--source=kubernetes.summary_api:''\"],\"image\":\"gcr.io/google_containers/metrics-server-amd64:v0.2.1\",\"name\":\"metrics-server\",\"ports\":[{\"containerPort\":443,\"name\":\"https\",\"protocol\":\"TCP\"}]},{\"command\":[\"/pod_nanny\",\"--config-dir=/etc/config\",\"--cpu=40m\",\"--extra-cpu=0.5m\",\"--memory=40Mi\",\"--extra-memory=4Mi\",\"--threshold=5\",\"--deployment=metrics-server-v0.2.1\",\"--container=metrics-server\",\"--poll-period=300000\",\"--estimator=exponential\"],\"env\":[{\"name\":\"MY_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"MY_POD_NAMESPACE\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.namespace\"}}}],\"image\":\"gcr.io/google_containers/addon-resizer:1.8.1\",\"name\":\"metrics-server-nanny\",\"resources\":{\"limits\":{\"cpu\":\"100m\",\"memory\":\"300Mi\"},\"requests\":{\"cpu\":\"5m\",\"memory\":\"50Mi\"}},\"volumeMounts\":[{\"mountPath\":\"/etc/config\",\"name\":\"metrics-server-config-volume\"}]}],\"serviceAccountName\":\"metrics-server\",\"tolerations\":[{\"key\":\"CriticalAddonsOnly\",\"operator\":\"Exists\"}],\"volumes\":[{\"configMap\":{\"name\":\"metrics-server-config\"},\"name\":\"metrics-server-config-volume\"}]}}}}\n"
},
"creationTimestamp": "2018-09-20T13:04:03Z",
"generation": 14,
"labels": {
"addonmanager.kubernetes.io/mode": "Reconcile",
"k8s-app": "metrics-server",
"kubernetes.io/cluster-service": "true",
"version": "v0.2.1"
},
"name": "metrics-server-v0.2.1",
"namespace": "kube-system",
"resourceVersion": "822513",
"selfLink": "/apis/extensions/v1beta1/namespaces/kube-system/deployments/metrics-server-v0.2.1",
"uid": "a5cd1f4c-bcd5-11e8-9313-42010a80005f"
},
"spec": {
"replicas": 1,
"selector": {
"matchLabels": {
"k8s-app": "metrics-server",
"version": "v0.2.1"
}
},
"strategy": {
"rollingUpdate": {
"maxSurge": 1,
"maxUnavailable": 1
},
"type": "RollingUpdate"
},
"template": {
"metadata": {
"annotations": {
"scheduler.alpha.kubernetes.io/critical-pod": ""
},
"creationTimestamp": null,
"labels": {
"k8s-app": "metrics-server",
"version": "v0.2.1"
},
"name": "metrics-server"
},
"spec": {
"containers": [
{
"command": [
"/metrics-server",
"--source=kubernetes.summary_api:''"
],
"image": "gcr.io/google_containers/metrics-server-amd64:v0.2.1",
"imagePullPolicy": "IfNotPresent",
"name": "metrics-server",
"ports": [
{
"containerPort": 443,
"name": "https",
"protocol": "TCP"
}
],
"resources": {
"limits": {
"cpu": "48m",
"memory": "104Mi"
},
"requests": {
"cpu": "48m",
"memory": "104Mi"
}
},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File"
},
{
"command": [
"/pod_nanny",
"--config-dir=/etc/config",
"--cpu=40m",
"--extra-cpu=0.5m",
"--memory=40Mi",
"--extra-memory=4Mi",
"--threshold=5",
"--deployment=metrics-server-v0.2.1",
"--container=metrics-server",
"--poll-period=300000",
"--estimator=exponential"
],
"env": [
{
"name": "MY_POD_NAME",
"valueFrom": {
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.name"
}
}
},
{
"name": "MY_POD_NAMESPACE",
"valueFrom": {
{
"apiVersion": "v1",
"fieldPath": "metadata.namespace"
}
}
}
],
"image": "gcr.io/google_containers/addon-resizer:1.8.1",
"imagePullPolicy": "IfNotPresent",
"name": "metrics-server-nanny",
"resources": {
"limits": {
"cpu": "100m",
"memory": "300Mi"
},
"requests": {
"cpu": "5m",
"memory": "50Mi"
}
},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"volumeMounts": [
{
"mountPath": "/etc/config",
"name": "metrics-server-config-volume"
}
]
}
],
"dnsPolicy": "ClusterFirst",
"restartPolicy": "Always",
"schedulerName": "default-scheduler",
"securityContext": {},
"serviceAccount": "metrics-server",
"serviceAccountName": "metrics-server",
"terminationGracePeriodSeconds": 30,
"tolerations": [
{
"key": "CriticalAddonsOnly",
"operator": "Exists"
}
],
"volumes": [
{
"configMap": {
"defaultMode": 420,
"name": "metrics-server-config"
},
"name": "metrics-server-config-volume"
}
]
}
}
},
"status": {
"availableReplicas": 1,
"conditions": [
{
"lastTransitionTime": "2018-09-20T13:04:03Z",
"lastUpdateTime": "2018-09-20T13:04:03Z",
"message": "Deployment has minimum availability.",
"reason": "MinimumReplicasAvailable",
"status": "True",
"type": "Available"
}
],
"observedGeneration": 14,
"readyReplicas": 1,
"replicas": 1,
"updatedReplicas": 1
}
}
Editing the yaml/flags of anything in kube-system on GKE (Google Kubernetes Engine) will not work as it will get reverted by the master. So, that part is working as intended.
It looks like fluentd which is auto-managed by GKE for logging, is what is causing the changes to get reverted. So the option I can think of, would be to disable the gke addons (ie cloud logging), and roll your own fluentd daemonset, and then configure things yourself. I will recommend you to visit this discussion for more information
Additionally, I will request you to take a look into this guide, if you'd like to roll your own fluentd on your cluster as well.
I encounter a trouble with the last version of kubernetes (1.5.1). I have a quiet non usual setup composed with 5 Redhat Enterprise server. 3 are nodes, 2 are masters. Both masters are on an etcd cluster, flannel had been also added in baremetal.
I have this looping log in the kube-DNS container :
Failed to list *api.Endpoints: Get https://*.*.*.33:443/api/v1/endpoints?resourceVersion=0: x509: failed to load system roots and no roots provided
I made a big number of tests concerning the certificate. Curl works with the same credentials perfectly. The generation has been made with the official recommandation of kubernetes.
This is my different files of configuration ( with just the censorship of the ip and hostname if needed).
kube-apiserver.yml
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "kube-apiserver",
"namespace": "kube-system",
"labels": {
"component": "kube-apiserver",
"tier": "control-plane"
}
},
"spec": {
"volumes": [
{
"name": "certs",
"hostPath": {
"path": "/etc/ssl/certs"
}
},
{
"name": "pki",
"hostPath": {
"path": "/etc/kubernetes"
}
}
],
"containers": [
{
"name": "kube-apiserver",
"image": "gcr.io/google_containers/kube-apiserver-amd64:v1.5.1",
"command": [
"/usr/local/bin/kube-apiserver",
"--v=0",
"--insecure-bind-address=127.0.0.1",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota",
"--service-cluster-ip-range=100.64.0.0/12",
"--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--client-ca-file=/etc/kubernetes/pki/ca.pem",
"--tls-cert-file=/etc/kubernetes/pki/apiserver.pem",
"--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--secure-port=5443",
"--allow-privileged",
"--advertise-address=X.X.X.33",
"--etcd-servers=http://X.X.X.33:2379,http://X.X.X.37:2379",
"--kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP"
],
"resources": {
"requests": {
"cpu": "250m"
}
},
"volumeMounts": [
{
"name": "certs",
"mountPath": "/etc/ssl/certs"
},
{
"name": "pki",
"readOnly": true,
"mountPath": "/etc/kubernetes/"
}
],
"livenessProbe": {
"httpGet": {
"path": "/healthz",
"port": 8080,
"host": "127.0.0.1"
},
"initialDelaySeconds": 15,
"timeoutSeconds": 15
}
}
],
"hostNetwork": true
}
}
kube-controlleur-manager.yml
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "kube-controller-manager",
"namespace": "kube-system",
"labels": {
"component": "kube-controller-manager",
"tier": "control-plane"
}
},
"spec": {
"volumes": [
{
"name": "pki",
"hostPath": {
"path": "/etc/kubernetes"
}
}
],
"containers": [
{
"name": "kube-controller-manager",
"image": "gcr.io/google_containers/kube-controller-manager-amd64:v1.5.1",
"command": [
"/usr/local/bin/kube-controller-manager",
"--v=0",
"--address=127.0.0.1",
"--leader-elect=true",
"--master=https://X.X.X.33",
"--cluster-name= kubernetes",
"--kubeconfig=/etc/kubernetes/kubeadminconfig",
"--root-ca-file=/etc/kubernetes/pki/ca.pem",
"--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem",
"--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem"
],
"resources": {
"requests": {
"cpu": "200m"
}
},
"volumeMounts": [
{
"name": "pki",
"readOnly": true,
"mountPath": "/etc/kubernetes/"
}
],
"livenessProbe": {
"httpGet": {
"path": "/healthz",
"port": 10252,
"host": "127.0.0.1"
},
"initialDelaySeconds": 15,
"timeoutSeconds": 15
}
}
],
"hostNetwork": true
}
}
kube-scheduler.yml
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "kube-scheduler",
"namespace": "kube-system",
"labels": {
"component": "kube-scheduler",
"tier": "control-plane"
}
},
"spec": {
"volumes": [
{
"name": "pki",
"hostPath": {
"path": "/etc/kubernetes"
}
}
],
"containers": [
{
"name": "kube-scheduler",
"image": "gcr.io/google_containers/kube-scheduler-amd64:v1.5.1",
"command": [
"/usr/local/bin/kube-scheduler",
"--v=0",
"--address=127.0.0.1",
"--leader-elect=true",
"--kubeconfig=/etc/kubernetes/kubeadminconfig",
"--master=https://X.X.X.33"
],
"resources": {
"requests": {
"cpu": "100m"
}
},
"volumeMounts": [
{
"name": "pki",
"readOnly": true,
"mountPath": "/etc/kubernetes/"
}
],
"livenessProbe": {
"httpGet": {
"path": "/healthz",
"port": 10251,
"host": "127.0.0.1"
},
"initialDelaySeconds": 15,
"timeoutSeconds": 15
}
}
],
"hostNetwork": true
}
}
haproxy.yml
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "haproxy",
"namespace": "kube-system",
"labels": {
"component": "kube-apiserver",
"tier": "control-plane"
}
},
"spec": {
"volumes": [
{
"name": "vol",
"hostPath": {
"path": "/etc/haproxy/haproxy.cfg"
}
}
],
"containers": [
{
"name": "haproxy",
"image": "docker.io/haproxy:1.7",
"resources": {
"requests": {
"cpu": "250m"
}
},
"volumeMounts": [
{
"name": "vol",
"readOnly": true,
"mountPath": "/usr/local/etc/haproxy/haproxy.cfg"
}
]
}
],
"hostNetwork": true
}
}
kubelet.service
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service
[Service]
WorkingDirectory=/var/lib/kubelet
EnvironmentFile=/etc/kubernetes/kubelet ExecStart=/usr/bin/kubelet \
$KUBELET_ADDRESS \
$KUBELET_POD_INFRA_CONTAINER \
$KUBELET_ARGS \
$KUBE_LOGTOSTDERR \
$KUBE_ALLOW_PRIV \
$KUBELET_NETWORK_ARGS \
$KUBELET_DNS_ARGS
Restart=on-failure
[Install]
WantedBy=multi-user.target
kubelet
KUBELET_ADDRESS="--address=0.0.0.0 --port=10250"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"
KUBELET_ARGS="--kubeconfig=/etc/kubernetes/kubeadminconfig --require-kubeconfig=true --pod-manifest-path=/etc/kubernetes/manifests"
KUBE_LOGTOSTDERR="--logtostderr=true --v=9"
KUBE_ALLOW_PRIV="--allow-privileged=true"
KUBELET_DNS_ARGS="--cluster-dns=100.64.0.10 --cluster-domain=cluster.local"
kubadminconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority: /etc/kubernetes/pki/ca.pem
server: https://X.X.X.33
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: admin
name: admin#kubernetes
- context:
cluster: kubernetes
user: kubelet
name: kubelet#kubernetes
current-context: admin#kubernetes
kind: Config
users:
- name: admin
user:
client-certificate: /etc/kubernetes/pki/admin.pem
client-key: /etc/kubernetes/pki/admin-key.pem
I already have seen most of the question relative from far to close to this question in the internet so i hope someone will have a hint to debug this.
I can do a deploy like this, but cannot do it via command line.
I was looking at doing it like this
kubectl create -f kubernetes-rc.json
{
"kind": "ReplicationController",
"apiVersion": "v1",
"metadata": {
"name": "foo-frontend-rc",
"labels": {
"www": true
},
"namespace": "foo"
},
"spec": {
"replicas": 1,
"template": {
"metadata": {
"labels": {
"app": "foo-frontend"
}
},
"spec": {
"containers": [
{
"name": "foo-frontend",
"image": "gcr.io/atomic-griffin-130023/foo-frontend:b3fc862",
"ports": [
{
"containerPort": 3009,
"protocol": "TCP"
}
],
"imagePullPolicy": "IfNotPresent"
}
],
"restartPolicy": "Always",
"dnsPolicy": "ClusterFirst"
}
}
}
}
and
kubectl create -f kubernetes-service.json
{
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": "foo-frontend-service"
},
"spec": {
"selector": {
"app": "foo-frontend-rc"
},
"ports": [
{
"protocol": "TCP",
"port": 80,
"targetPort": 3009
}
]
}
}
to no avail. It creates the rc, but it won’t expose the service externally.
Your service's selector is wrong. It should be selecting a label from the pod template, not a label on the RC itself.
If you change the following in your service:
"selector": {
"app": "foo-frontend-rc"
},
to:
"selector": {
"app": "foo-frontend"
},
It should fix it.
Update
Change your service definition to
{
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": "foo-frontend-service"
},
"spec": {
"selector": {
"app": "foo-frontend"
},
"ports": [
{
"protocol": "TCP",
"port": 80,
"targetPort": 3009,
"nodePort": 30009
}
],
"type": "LoadBalancer"
}
}