Kubernetes pods are getting failed with volume attach issue - kubernetes

Question

We are seeing some of the pods getting failed while mounting/attaching volume to pods. this is happening intermittently and after bouncing kubelet service pods are able to reattach the volumes and succeeding. seeing the below error when pod get struck.
I created single PV and corresponding PVC to use volume mounts for pods:
Error:
Warning FailedMount 16m (x4 over 43m) kubelet, ca1md-k8w03-02 Unable to attach or mount volumes: unmounted volumes=[workdir], unattached volumes=[default-token-pwvpc podmetadata docker-sock workdir]: timed out waiting for the condition
Warning FailedMount 7m32s (x5 over 41m) kubelet, ca1md-k8w03-02 Unable to attach or mount volumes: unmounted volumes=[workdir], unattached volumes=[docker-sock workdir default-token-pwvpc podmetadata]: timed out waiting for the condition
Warning FailedMount 3m2s (x10 over 45m) kubelet, ca1md-k8w03-02 Unable to attach or mount volumes: unmounted volumes=[workdir], unattached volumes=[podmetadata docker-sock workdir default-token-pwvpc]: timed out waiting for the condition
Warning FailedMount 45s (x2 over 21m) kubelet, ca1md-k8w03-02 Unable to attach or mount volumes: unmounted volumes=[workdir], unattached volumes=[workdir default-token-pwvpc podmetadata docker-sock]: timed out waiting for the condition
Version:
Client Version: v1.17.2
Server Version: v1.17.2
Host OS:
Centos 7.7
CNI:
Weave
apiVersion: v1
items:
- apiVersion: apps/v1
kind: StatefulSet
metadata:
creationTimestamp: "2020-07-17T21:55:47Z"
generation: 1
labels:
logicmonitor.com/collectorset: kubernetes-03
managedFields:
- apiVersion: apps/v1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:labels:
.: {}
f:logicmonitor.com/collectorset: {}
f:spec:
f:podManagementPolicy: {}
f:replicas: {}
f:revisionHistoryLimit: {}
f:selector:
f:matchLabels:
.: {}
f:logicmonitor.com/collectorset: {}
f:template:
f:metadata:
f:labels:
.: {}
f:logicmonitor.com/collectorset: {}
f:namespace: {}
f:spec:
f:affinity:
.: {}
f:podAntiAffinity:
.: {}
f:requiredDuringSchedulingIgnoredDuringExecution: {}
f:containers:
k:{"name":"collector"}:
.: {}
f:env:
.: {}
k:{"name":"COLLECTOR_IDS"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"access_id"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
f:optional: {}
k:{"name":"access_key"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
f:optional: {}
k:{"name":"account"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
f:optional: {}
k:{"name":"collector_size"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"collector_version"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"kubernetes"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"use_ea"}:
.: {}
f:name: {}
f:value: {}
f:image: {}
f:imagePullPolicy: {}
f:name: {}
f:resources:
.: {}
f:limits:
.: {}
f:memory: {}
f:terminationMessagePath: {}
f:terminationMessagePolicy: {}
f:dnsPolicy: {}
f:restartPolicy: {}
f:schedulerName: {}
f:securityContext: {}
f:serviceAccount: {}
f:serviceAccountName: {}
f:terminationGracePeriodSeconds: {}
f:updateStrategy:
f:type: {}
f:status:
f:replicas: {}
manager: collectorset-controller
operation: Update
time: "2020-08-22T03:42:35Z"
name: kubernetes-03
namespace: default
resourceVersion: "10831902"
selfLink: /apis/apps/v1/namespaces/default/statefulsets/kubernetes-03
uid: 1296654b-77bc-4af8-9537-04f0a00bdd0c
spec:
podManagementPolicy: Parallel
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
logicmonitor.com/collectorset: kubernetes-03
serviceName: ""
template:
metadata:
creationTimestamp: null
labels:
logicmonitor.com/collectorset: kubernetes-03
namespace: default
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
logicmonitor.com/collectorset: kubernetes-03
topologyKey: kubernetes.io/hostname
containers:
- env:
- name: account
valueFrom:
secretKeyRef:
key: account
name: collectorset-controller
optional: false
- name: access_id
valueFrom:
secretKeyRef:
key: accessID
name: collectorset-controller
optional: false
- name: access_key
valueFrom:
secretKeyRef:
key: accessKey
name: collectorset-controller
optional: false
- name: kubernetes
value: "true"
- name: collector_size
value: small
- name: collector_version
value: "0"
- name: use_ea
value: "false"
- name: COLLECTOR_IDS
value: "205"
image: logicmonitor/collector:latest
imagePullPolicy: Always
name: collector
resources:
limits:
memory: 2Gi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: collector
serviceAccountName: collector
terminationGracePeriodSeconds: 30
updateStrategy:
type: RollingUpdate
status:
collisionCount: 0
currentReplicas: 1
currentRevision: kubernetes-03-655b46ff69
observedGeneration: 1
readyReplicas: 1
replicas: 1
updateRevision: kubernetes-03-655b46ff69
updatedReplicas: 1
kind: List
metadata:
resourceVersion: ""
selfLink: ""
kubectl get pv -o yaml
apiVersion: v1
items:
- apiVersion: v1
kind: PersistentVolume
metadata:
annotations:
pv.kubernetes.io/bound-by-controller: "yes"
creationTimestamp: "2020-07-10T00:21:49Z"
finalizers:
- kubernetes.io/pv-protection
labels:
type: local
managedFields:
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:labels:
.: {}
f:type: {}
f:spec:
f:accessModes: {}
f:capacity:
.: {}
f:storage: {}
f:nfs:
.: {}
f:path: {}
f:server: {}
f:persistentVolumeReclaimPolicy: {}
f:storageClassName: {}
f:volumeMode: {}
manager: kubectl
operation: Update
time: "2020-07-10T00:21:49Z"
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:pv.kubernetes.io/bound-by-controller: {}
f:spec:
f:claimRef:
.: {}
f:apiVersion: {}
f:kind: {}
f:name: {}
f:namespace: {}
f:resourceVersion: {}
f:uid: {}
f:status:
f:phase: {}
manager: kube-controller-manager
operation: Update
time: "2020-07-10T00:21:52Z"
name: pv-modeldata
resourceVersion: "15764"
selfLink: /api/v1/persistentvolumes/pv-data
uid: 68521e84-0aa9-4643-a047-441a61451599
spec:
accessModes:
- ReadWriteMany
capacity:
storage: 5999Gi
claimRef:
apiVersion: v1
kind: PersistentVolumeClaim
name: pvc-modeldata
namespace: default
resourceVersion: "15762"
uid: e4a1309e-339d-4ed5-8fe1-9ed32f779ea7
nfs:
path: /k8fs03
server: kubemaster01.rms.com
persistentVolumeReclaimPolicy: Retain
storageClassName: manual
volumeMode: Filesystem
status:
phase: Bound
kind: List
metadata:
resourceVersion: ""
selfLink: ""
# kubectl get pvc -o yaml
apiVersion: v1
items:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
pv.kubernetes.io/bind-completed: "yes"
pv.kubernetes.io/bound-by-controller: "yes"
creationTimestamp: "2020-07-10T00:21:52Z"
finalizers:
- kubernetes.io/pvc-protection
managedFields:
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:pv.kubernetes.io/bind-completed: {}
f:pv.kubernetes.io/bound-by-controller: {}
f:spec:
f:volumeName: {}
f:status:
f:accessModes: {}
f:capacity:
.: {}
f:storage: {}
f:phase: {}
manager: kube-controller-manager
operation: Update
time: "2020-07-10T00:21:52Z"
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:spec:
f:accessModes: {}
f:resources:
f:requests:
.: {}
f:storage: {}
f:storageClassName: {}
f:volumeMode: {}
manager: kubectl
operation: Update
time: "2020-07-10T00:21:52Z"
name: pvc-modeldata
namespace: default
resourceVersion: "15766"
selfLink: /api/v1/namespaces/default/persistentvolumeclaims/pvc-data
uid: e4a1309e-339d-4ed5-8fe1-9ed32f779ea7
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 5999Gi
storageClassName: manual
volumeMode: Filesystem
volumeName: pv-data
status:
accessModes:
- ReadWriteMany
capacity:
storage: 5999Gi
phase: Bound
kind: List
metadata:
resourceVersion: ""
selfLink: ""
# kubectl get sc -o yaml
apiVersion: v1
items: []
kind: List
metadata:
resourceVersion: ""
selfLink: ""
Thanks,
Chittu