Related
I want to rm -rf /var/run/secrets/kubernetes.io/serviceaccount/ to delete the default Kubernetes service account for testing anonymous API access.
However, running the above command shows that many of the files are on a read-only filesystem, so I want to temporarily remount the filesystem (mount -o remount) to delete the files.
Now, how can I see which filesystem in the output of mount to remount? None of the filesystems below are mounted on a /var/run/ path. The closest match is /var/lib/ in the options for the overlay filesystem mounted on /.
How can I safely delete the files under /var/run/secrets/kubernetes.io/serviceaccount/?
root#ctf1-deploy1-6fd44cbcd6-vrckg:~# rm -rf /var/run/secrets/kubernetes.io/serviceaccount/
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/..data': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/token': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/namespace': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/..2020_03_25_08_59_25.631059710/namespace': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/..2020_03_25_08_59_25.631059710/ca.crt': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/..2020_03_25_08_59_25.631059710/token': Read-only file system
root#ctf1-deploy1-6fd44cbcd6-vrckg:~# mount
overlay on / type overlay (rw,relatime,lowerdir=/var/lib/docker/overlay2/l/ZLOIVO6AXDQFZ3NU3O4VYG5QJC:/var/lib/docker/overlay2/l/MZY5Y3MJC6IVDUFSNOQEU55JZJ:/var/lib/docker/overlay2/l/E5HAR5VEWTG6MCFYN22KDJNMK3:/var/lib/docker/overlay2/l/5Z2WGKVJRNGPXV5QIFR5CWHJSE:/var/lib/docker/overlay2/l/U5HNVHXGGWRIGBX3XJV5CW5VIZ:/var/lib/docker/overlay2/l/TI5WPYBQSWQJXBMOY7DT5DN26Z:/var/lib/docker/overlay2/l/XOZNIEPFIZTLP2HEHFKL66H4EO,upperdir=/var/lib/docker/overlay2/4ea51426b9eb47af0faf53e60a25b6cffae64c3338130663efd1068c7d2ffb20/diff,workdir=/var/lib/docker/overlay2/4ea51426b9eb47af0faf53e60a25b6cffae64c3338130663efd1068c7d2ffb20/work)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev type tmpfs (rw,nosuid,size=65536k,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=666)
sysfs on /sys type sysfs (ro,nosuid,nodev,noexec,relatime)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,relatime,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (ro,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (ro,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/blkio type cgroup (ro,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (ro,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/perf_event type cgroup (ro,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/devices type cgroup (ro,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/memory type cgroup (ro,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/freezer type cgroup (ro,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/cpuset type cgroup (ro,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/pids type cgroup (ro,nosuid,nodev,noexec,relatime,pids)
mqueue on /dev/mqueue type mqueue (rw,nosuid,nodev,noexec,relatime)
/dev/nvme0n1p2 on /dev/termination-log type ext4 (rw,relatime,data=ordered)
/dev/nvme0n1p2 on /etc/resolv.conf type ext4 (rw,relatime,data=ordered)
/dev/nvme0n1p2 on /etc/hostname type ext4 (rw,relatime,data=ordered)
/dev/nvme0n1p2 on /etc/hosts type ext4 (rw,relatime,data=ordered)
shm on /dev/shm type tmpfs (rw,nosuid,nodev,noexec,relatime,size=65536k)
tmpfs on /run/secrets/kubernetes.io/serviceaccount type tmpfs (ro,relatime)
proc on /proc/bus type proc (ro,relatime)
proc on /proc/fs type proc (ro,relatime)
proc on /proc/irq type proc (ro,relatime)
proc on /proc/sys type proc (ro,relatime)
proc on /proc/sysrq-trigger type proc (ro,relatime)
tmpfs on /proc/acpi type tmpfs (ro,relatime)
tmpfs on /proc/kcore type tmpfs (rw,nosuid,size=65536k,mode=755)
tmpfs on /proc/keys type tmpfs (rw,nosuid,size=65536k,mode=755)
tmpfs on /proc/timer_list type tmpfs (rw,nosuid,size=65536k,mode=755)
tmpfs on /proc/sched_debug type tmpfs (rw,nosuid,size=65536k,mode=755)
tmpfs on /sys/firmware type tmpfs (ro,relatime)
I'm running debian 9.6. with uboot on beaglebone black, 32bit ARM system.
Image downloaded from:
https://rcn-ee.com/rootfs/bb.org/testing/2019-01-06/stretch-console/bone-debian-9.6-console-armhf-2019-01-06-1gb.img.xz
Kernel command params:
Jan 17 14:19:19 bbb-test kernel: Kernel command line: console=ttyO0,115200n8 bone_capemgr.enable_partno=BB-UA
RT1,BB-UART4,BB-UART5 bone_capemgr.uboot_capemgr_enabled=1 root=/dev/mmcblk1p1 ro rootfstype=ext4 rootwait coherent_p
ool=1M net.ifnames=0 quiet cape_universal=enable cgroup_enable=cpuset cgroup_enable=memory cgroup_memory=1 swapaccoun
t=1
debian#bbb-test:~$ uname -a
Linux bbb-test 4.14.79-ti-r84 #1 SMP PREEMPT Tue Nov 13 20:11:18 UTC 2018 armv7l GNU/Linux
Docker is installed and working on this machine. Kubelet log:
debian#bbb-test:~$ sudo kubelet
I0117 14:31:25.972837 2841 server.go:407] Version: v1.13.2
I0117 14:31:25.982041 2841 plugins.go:103] No cloud provider specified.
W0117 14:31:25.995051 2841 server.go:552] standalone mode, no API client
E0117 14:31:26.621471 2841 machine.go:194] failed to get cache information for node 0: open /sys/devices/system/cpu/cpu0/cache: no such file or directory
W0117 14:31:26.655651 2841 server.go:464] No api server defined - no events will be sent to API server.
I0117 14:31:26.657953 2841 server.go:666] --cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /
I0117 14:31:26.664398 2841 container_manager_linux.go:248] container manager verified user specified cgroup-root exists: []
I0117 14:31:26.666142 2841 container_manager_linux.go:253] Creating Container Manager object based on Node Config: {RuntimeCgroupsName: SystemCgroupsName: KubeletCgroupsName: ContainerRuntime:docker CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:cgroupfs KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:false NodeAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionThresholds:[{Signal:memory.available Operator:LessThan Value:{Quantity:100Mi Percentage:0} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.1} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.inodesFree Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.15} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] ExperimentalCPUManagerPolicy:none ExperimentalCPUManagerReconcilePeriod:10s ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms}
I0117 14:31:26.675196 2841 container_manager_linux.go:272] Creating device plugin manager: true
I0117 14:31:26.676961 2841 state_mem.go:36] [cpumanager] initializing new in-memory state store
I0117 14:31:26.680692 2841 state_mem.go:84] [cpumanager] updated default cpuset: ""
I0117 14:31:26.682789 2841 state_mem.go:92] [cpumanager] updated cpuset assignments: "map[]"
I0117 14:31:26.760365 2841 client.go:75] Connecting to docker on unix:///var/run/docker.sock
I0117 14:31:26.762342 2841 client.go:104] Start docker client with request timeout=2m0s
W0117 14:31:26.820114 2841 docker_service.go:540] Hairpin mode set to "promiscuous-bridge" but kubenet is not enabled, falling back to "hairpin-veth"
I0117 14:31:26.821450 2841 docker_service.go:236] Hairpin mode set to "hairpin-veth"
W0117 14:31:26.832217 2841 cni.go:203] Unable to update cni config: No networks found in /etc/cni/net.d
W0117 14:31:26.932660 2841 hostport_manager.go:68] The binary conntrack is not installed, this can cause failures in network connection cleanup.
I0117 14:31:26.980052 2841 docker_service.go:251] Docker cri networking managed by kubernetes.io/no-op
I0117 14:31:27.241438 2841 docker_service.go:256] Docker Info: &{ID:YHXC:3CJD:MPM6:SQ6I:37PA:E7CY:YG32:EQZH:XKFS:5FLV:OHRN:UYQS Containers:0 ContainersRunning:0 ContainersPaused:0 ContainersStopped:0 Images:0 Driver:overlay2 DriverStatus:[[Backing Filesystem extfs] [Supports d_type true] [Native Overlay Diff true]] SystemStatus:[] Plugins:{Volume:[local] Network:[bridge host macvlan null overlay] Authorization:[] Log:[awslogs fluentd gcplogs gelf journald json-file logentries splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:true CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:false IPv4Forwarding:true BridgeNfIptables:true BridgeNfIP6tables:true Debug:false NFd:24 OomKillDisable:true NGoroutines:44 SystemTime:2019-01-17T14:31:27.033459842Z LoggingDriver:json-file CgroupDriver:cgroupfs NEventsListener:0 KernelVersion:4.14.79-ti-r84 OperatingSystem:Debian GNU/Linux 9 (stretch) OSType:linux Architecture:armv7l IndexServerAddress:https://index.docker.io/v1/ RegistryConfig:0x65be080 NCPU:1 MemTotal:506748928 GenericResources:[] DockerRootDir:/var/lib/docker HTTPProxy: HTTPSProxy: NoProxy: Name:bbb-test Labels:[] ExperimentalBuild:false ServerVersion:18.06.1-ce ClusterStore: ClusterAdvertise: Runtimes:map[runc:{Path:docker-runc Args:[]}] DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:[] Nodes:0 Managers:0 Cluster:<nil>} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:468a545b9edcd5932818eb9de8e72413e616e86e Expected:468a545b9edcd5932818eb9de8e72413e616e86e} RuncCommit:{ID:69663f0bd4b60df09991c08812a60108003fa340 Expected:69663f0bd4b60df09991c08812a60108003fa340} InitCommit:{ID:fec3683 Expected:fec3683} SecurityOptions:[name=seccomp,profile=default]}
I0117 14:31:27.248202 2841 docker_service.go:269] Setting cgroupDriver to cgroupfs
I0117 14:31:27.558049 2841 kuberuntime_manager.go:198] Container runtime docker initialized, version: 18.06.1-ce, apiVersion: 1.38.0
I0117 14:31:27.611546 2841 server.go:999] Started kubelet
E0117 14:31:27.623092 2841 kubelet.go:1308] Image garbage collection failed once. Stats initialization may not have completed yet: failed to get imageFs info: unable to find data in memory cache
W0117 14:31:27.623253 2841 kubelet.go:1412] No api server defined - no node status update will be sent.
I0117 14:31:27.644045 2841 fs_resource_analyzer.go:66] Starting FS ResourceAnalyzer
I0117 14:31:27.650728 2841 status_manager.go:148] Kubernetes client is nil, not starting status manager.
I0117 14:31:27.651717 2841 kubelet.go:1829] Starting kubelet main sync loop.
I0117 14:31:27.652815 2841 kubelet.go:1846] skipping pod synchronization - [container runtime status check may not have completed yet PLEG is not healthy: pleg has yet to be successful]
I0117 14:31:27.623347 2841 server.go:137] Starting to listen on 0.0.0.0:10250
I0117 14:31:27.671780 2841 server.go:333] Adding debug handlers to kubelet server.
I0117 14:31:27.684102 2841 volume_manager.go:248] Starting Kubelet Volume Manager
I0117 14:31:27.713834 2841 desired_state_of_world_populator.go:130] Desired state populator starts to run
I0117 14:31:27.808299 2841 kubelet.go:1846] skipping pod synchronization - [container runtime status check may not have completed yet PLEG is not healthy: pleg has yet to be successful]
I0117 14:31:27.943782 2841 reconciler.go:154] Reconciler: start to sync state
I0117 14:31:28.051598 2841 kubelet.go:1846] skipping pod synchronization - [container runtime status check may not have completed yet]
W0117 14:31:28.415337 2841 nvidia.go:66] Error reading "/sys/bus/pci/devices/": open /sys/bus/pci/devices/: no such file or directory
I0117 14:31:28.496333 2841 kubelet.go:1846] skipping pod synchronization - [container runtime status check may not have completed yet]
I0117 14:31:29.362011 2841 kubelet.go:1846] skipping pod synchronization - [container runtime status check may not have completed yet]
W0117 14:31:29.477526 2841 container.go:409] Failed to create summary reader for "/system.slice/system-openvpn.slice/openvpn#bbb.service": none of the resources are being tracked.
I0117 14:31:30.999869 2841 kubelet.go:1846] skipping pod synchronization - [container runtime status check may not have completed yet]
I0117 14:31:31.131276 2841 kubelet_node_status.go:278] Setting node annotation to enable volume controller attach/detach
I0117 14:31:31.184490 2841 cpu_manager.go:155] [cpumanager] starting with none policy
I0117 14:31:31.195065 2841 cpu_manager.go:156] [cpumanager] reconciling every 10s
I0117 14:31:31.196076 2841 policy_none.go:42] [cpumanager] none policy: Start
F0117 14:31:31.199910 2841 kubelet.go:1384] Failed to start ContainerManager system validation failed - Following Cgroup subsystem not mounted: [cpuset]
d
The issue persists with valid config and proper kubeconfig.
EDIT: additional data
cgroups are kinda mounted:
debian#bbb-test:~$ mount
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
udev on /dev type devtmpfs (rw,nosuid,relatime,size=220140k,nr_inodes=55035,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=49492k,mode=755)
/dev/mmcblk1p1 on / type ext4 (rw,noatime,errors=remount-ro,data=ordered)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=34,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=15992)
mqueue on /dev/mqueue type mqueue (rw,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime)
configfs on /sys/kernel/config type configfs (rw,relatime)
tmpfs on /run/user/1000 type tmpfs (rw,nosuid,nodev,relatime,size=49488k,mode=700,uid=1000,gid=1000)
debian#bbb-test:~$ systemctl status 'cg*'
debian#bbb-test:~$
I've updated to 4.19 kernel and full system package upgrade and it resolved the problem. That's kinda good enough for me
I was facing the same error on Ubuntu 20.04 (kernel 5.14), Docker 20.10.17, minikube 1.26.1:
Failed to start ContainerManager system validation failed - Following Cgroup subsystem not mounted: [cpuset]
Note of the solutions above (or this, or this) worked for me.
Eventually, I figured out it had to do with Docker being configured to use a rootless context. docker context use default fixed it.
I suggest looking at https://unix.stackexchange.com/questions/427327/how-can-i-check-if-cgroups-are-available-on-my-linux-host to check if your cgroup activation works as expected.
cgroup_enable=cpuset as a parameter for kernel should be enough unless there's something else that's missing (eg: maybe it's in the wrong place?)
Our server runs Centos 7.2, today it seems to fail.
When I connect to it using ssh, I get
-bash: /share/home/MGI/.bash_profile: Input/output error
and log in like this:
-bash-4.2$
ls failed,
ls: cannot open directory .: Input/output error
Trying to edit a file with vi will get Permission denied
but cd and pwd is still of use.
I googled it and found the disk might be damaged, then I tried some suggestions. mount give me this:
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
devtmpfs on /dev type devtmpfs (rw,nosuid,size=32831312k,nr_inodes=8207828,mode=755)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/net_cls type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
configfs on /sys/kernel/config type configfs (rw,relatime)
/dev/md126p2 on / type ext4 (rw,relatime,data=ordered)
mqueue on /dev/mqueue type mqueue (rw,relatime)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=35,pgrp=1,timeout=300,minproto=5,maxproto=5,direct)
sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime)
nfsd on /proc/fs/nfsd type nfsd (rw,relatime)
/dev/mapper/mpatha1 on /share type xfs (rw,relatime,attr2,inode64,noquota)
/dev/md126p1 on /boot type ext4 (rw,relatime,data=ordered)
fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime)
tmpfs on /run/user/1038 type tmpfs (rw,nosuid,nodev,relatime,size=6569364k,mode=700,uid=1038,gid=1039)
tmpfs on /run/user/1016 type tmpfs (rw,nosuid,nodev,relatime,size=6569364k,mode=700,uid=1016,gid=1016)
tmpfs on /run/user/1008 type tmpfs (rw,nosuid,nodev,relatime,size=6569364k,mode=700,uid=1008,gid=1008)
tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=6569364k,mode=700)
gvfsd-fuse on /run/user/0/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=0,group_id=0)
tmpfs on /run/user/1019 type tmpfs (rw,nosuid,nodev,relatime,size=6569364k,mode=700,uid=1019,gid=1020)
dmesg gives a long output and these two lines appear frequently:
XFS (dm-1): xfs_log_force: error -5 returned.
scsi 11:0:0:0: alua: rtpg failed with 8000002
What should I do now to find out what is actually going wrong and how can I fix it? Many thanks.
I'm running kubernetes on bare-metal Debian (3 masters, 2 workers, PoC for now). I followed k8s-the-hard-way, and I'm running into the following problem on my kubelet:
Failed to get system container stats for
"/system.slice/docker.service": failed to get cgroup stats for
"/system.slice/docker.service": failed to get cgroup stats for
"/system.slice/docker.service": failed to get container info for
"/system.slice/docker.service": unknown container
"/system.slice/docker.service"
And I have the same message for kubelet.service.
I have some files about those cgroups:
$ ls /sys/fs/cgroup/systemd/system.slice/docker.service
cgroup.clone_children cgroup.procs notify_on_release tasks
$ ls /sys/fs/cgroup/systemd/system.slice/kubelet.service/
cgroup.clone_children cgroup.procs notify_on_release tasks
And cadvisor tells me:
$ curl http://127.0.0.1:4194/validate
cAdvisor version:
OS version: Debian GNU/Linux 8 (jessie)
Kernel version: [Supported and recommended]
Kernel version is 3.16.0-4-amd64. Versions >= 2.6 are supported. 3.0+ are recommended.
Cgroup setup: [Supported and recommended]
Available cgroups: map[cpu:1 memory:1 freezer:1 net_prio:1 cpuset:1 cpuacct:1 devices:1 net_cls:1 blkio:1 perf_event:1]
Following cgroups are required: [cpu cpuacct]
Following other cgroups are recommended: [memory blkio cpuset devices freezer]
Hierarchical memory accounting enabled. Reported memory usage includes memory used by child containers.
Cgroup mount setup: [Supported and recommended]
Cgroups are mounted at /sys/fs/cgroup.
Cgroup mount directories: blkio cpu cpu,cpuacct cpuacct cpuset devices freezer memory net_cls net_cls,net_prio net_prio perf_event systemd
Any cgroup mount point that is detectible and accessible is supported. /sys/fs/cgroup is recommended as a standard location.
Cgroup mounts:
cgroup /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd 0 0
cgroup /sys/fs/cgroup/cpuset cgroup rw,nosuid,nodev,noexec,relatime,cpuset 0 0
cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0
cgroup /sys/fs/cgroup/memory cgroup rw,nosuid,nodev,noexec,relatime,memory 0 0
cgroup /sys/fs/cgroup/devices cgroup rw,nosuid,nodev,noexec,relatime,devices 0 0
cgroup /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0
cgroup /sys/fs/cgroup/net_cls,net_prio cgroup rw,nosuid,nodev,noexec,relatime,net_cls,net_prio 0 0
cgroup /sys/fs/cgroup/blkio cgroup rw,nosuid,nodev,noexec,relatime,blkio 0 0
cgroup /sys/fs/cgroup/perf_event cgroup rw,nosuid,nodev,noexec,relatime,perf_event 0 0
Managed containers:
/kubepods/burstable/pod76099b4b-af57-11e7-9b82-fa163ea0076a
/kubepods/besteffort/pod6ed4ee49-af53-11e7-9b82-fa163ea0076a/f9da6bf60a186c47bd704bbe3cc18b25d07d4e7034d185341a090dc3519c047a
Namespace: docker
Aliases:
k8s_tiller_tiller-deploy-cffb976df-5s6np_kube-system_6ed4ee49-af53-11e7-9b82-fa163ea0076a_1
f9da6bf60a186c47bd704bbe3cc18b25d07d4e7034d185341a090dc3519c047a
/kubepods/burstable/pod76099b4b-af57-11e7-9b82-fa163ea0076a/956911118c342375abfb7a07ec3bb37451bbc64a1e141321b6284cf5049e385f
EDIT
Disabling the cadvisor port on kubelet (--cadvisor-port=0) doesn't fix that.
Try to start kubelet with
--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice
I'm using this solution on RHEL7 with Kubelet 1.8.0 and Docker 1.12
The angeloxx's workaround works also on AWS default image for kops (k8s-1.8-debian-jessie-amd64-hvm-ebs-2017-12-02 (ami-bd229ec4))
sudo vim /etc/sysconfig/kubelet
add at the end of DAEMON_ARGS string:
--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice
finally:
sudo systemctl restart kubelet
I had to do a yum update in addition to this change to make it work. Might be helpful for others trying this workaround.
Thanks angeloxx!
I'm following the kubernetes guide:
https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/setup-ha-etcd-with-kubeadm/
In the instructions, they have you make a file:
/usr/lib/systemd/system/kubelet.service.d/20-etcd-service-manager.conf
with the line:
ExecStart=/usr/bin/kubelet --address=127.0.0.1 --pod-manifest-path=/etc/kubernetes/manifests --cgroup-driver=systemd
I took your answer and added it to the end of the ExecStart line:
ExecStart=/usr/bin/kubelet --address=127.0.0.1 --pod-manifest-path=/etc/kubernetes/manifests --cgroup-driver=systemd --runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice
I'm writing this in case it helps someone else
# wolmi Thanks for the edit!
One more note:
The config I have above is for my etcd cluster, NOT the kubernetes nodes. A file like 20-etcd-service-manager.conf on a node would override all the settings in the "10-kubeadm.conf" file, causing all kinds if missed configurations. Use the "/var/lib/kubelet/config.yaml" file for nodes and/or /var/lib/kubelet/kubeadm-flags.env.
For those a little further along, in kops AMI kope.io/k8s-1.8-debian-jessie-amd64-hvm-ebs-2018-02-08 as above, I had to add:
add at the end of DAEMON_ARGS string:
--runtime-cgroups=/lib/systemd/system/kubelet.service --kubelet-cgroups=/lib/systemd/system/kubelet.service
and then:
sudo systemctl restart kubelet
but I found I was still getting:
Failed to get system container stats for "/system.slice/docker.service": failed to get cgroup stats for "/system.slice/docker.service": failed to get container info for "/system.slice/docker.service": unknown container "/system.slice/docker.service"
restarting dockerd resolved this error:
sudo systemctl restart docker
Thanks
After a little more digging around I found a better resolution to add this into the kops configuration:
https://github.com/kubernetes/kops/issues/4049
k8s 1.2 deployed locally, single-node docker
Am I doing something wrong? Is this working for everyone else or is something broken in my k8s deployment?
Following the example in the ConfigMaps guide, /etc/config/special.how should be created below but is not:
[root#totoro brs-kubernetes]# kubectl create -f example.yaml
configmap "special-config" created
pod "dapi-test-pod" created
[root#totoro brs-kubernetes]# kubectl exec -it dapi-test-pod -- sh
/ # cd /etc/config/
/etc/config # ls
/etc/config # ls -alh
total 4
drwxrwxrwt 2 root root 40 Mar 23 18:47 .
drwxr-xr-x 7 root root 4.0K Mar 23 18:47 ..
/etc/config #
example.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: special-config
namespace: default
data:
special.how: very
special.type: charm
---
apiVersion: v1
kind: Pod
metadata:
name: dapi-test-pod
spec:
containers:
- name: test-container
image: gcr.io/google_containers/busybox
command: ["sleep", "100"]
volumeMounts:
- name: config-volume
mountPath: /etc/config
volumes:
- name: config-volume
configMap:
name: special-config
items:
- key: special.how
path: how.file
restartPolicy: Never
Summary of conformance test failures follows (asked to run by jayunit100). Full run in this gist.
Summarizing 7 Failures:
[Fail] ConfigMap [It] updates should be reflected in volume [Conformance]
/home/schou/dev/kubernetes/_output/local/go/src/k8s.io/kubernetes/test/e2e/configmap.go:262
[Fail] Downward API volume [It] should provide podname only [Conformance]
/home/schou/dev/kubernetes/_output/local/go/src/k8s.io/kubernetes/test/e2e/util.go:1637
[Fail] Downward API volume [It] should update labels on modification [Conformance]
/home/schou/dev/kubernetes/_output/local/go/src/k8s.io/kubernetes/test/e2e/downwardapi_volume.go:82
[Fail] ConfigMap [It] should be consumable from pods in volume with mappings [Conformance]
/home/schou/dev/kubernetes/_output/local/go/src/k8s.io/kubernetes/test/e2e/util.go:1637
[Fail] Networking [It] should function for intra-pod communication [Conformance]
/home/schou/dev/kubernetes/_output/local/go/src/k8s.io/kubernetes/test/e2e/networking.go:121
[Fail] Downward API volume [It] should update annotations on modification [Conformance]
/home/schou/dev/kubernetes/_output/local/go/src/k8s.io/kubernetes/test/e2e/downwardapi_volume.go:119
[Fail] ConfigMap [It] should be consumable from pods in volume [Conformance]
/home/schou/dev/kubernetes/_output/local/go/src/k8s.io/kubernetes/test/e2e/util.go:1637
Ran 93 of 265 Specs in 2875.468 seconds
FAIL! -- 86 Passed | 7 Failed | 0 Pending | 172 Skipped --- FAIL: TestE2E (2875.48s)
FAIL
Output of findmnt:
[schou#totoro single-node]$ findmnt
TARGET SOURCE FSTYPE OPTIONS
/ /dev/mapper/fedora-root
│ ext4 rw,relatime,data=ordere
├─/sys sysfs sysfs rw,nosuid,nodev,noexec,
│ ├─/sys/kernel/security securityfs securit rw,nosuid,nodev,noexec,
│ ├─/sys/fs/cgroup tmpfs tmpfs ro,nosuid,nodev,noexec,
│ │ ├─/sys/fs/cgroup/systemd cgroup cgroup rw,nosuid,nodev,noexec,
│ │ ├─/sys/fs/cgroup/cpuset cgroup cgroup rw,nosuid,nodev,noexec,
│ │ ├─/sys/fs/cgroup/net_cls,net_prio cgroup cgroup rw,nosuid,nodev,noexec,
│ │ ├─/sys/fs/cgroup/memory cgroup cgroup rw,nosuid,nodev,noexec,
│ │ ├─/sys/fs/cgroup/hugetlb cgroup cgroup rw,nosuid,nodev,noexec,
│ │ ├─/sys/fs/cgroup/cpu,cpuacct cgroup cgroup rw,nosuid,nodev,noexec,
│ │ ├─/sys/fs/cgroup/perf_event cgroup cgroup rw,nosuid,nodev,noexec,
│ │ ├─/sys/fs/cgroup/pids cgroup cgroup rw,nosuid,nodev,noexec,
│ │ ├─/sys/fs/cgroup/blkio cgroup cgroup rw,nosuid,nodev,noexec,
│ │ ├─/sys/fs/cgroup/freezer cgroup cgroup rw,nosuid,nodev,noexec,
│ │ └─/sys/fs/cgroup/devices cgroup cgroup rw,nosuid,nodev,noexec,
│ ├─/sys/fs/pstore pstore pstore rw,nosuid,nodev,noexec,
│ ├─/sys/firmware/efi/efivars efivarfs efivarf rw,nosuid,nodev,noexec,
│ ├─/sys/kernel/debug debugfs debugfs rw,relatime
│ ├─/sys/kernel/config configfs configf rw,relatime
│ └─/sys/fs/fuse/connections fusectl fusectl rw,relatime
├─/proc proc proc rw,nosuid,nodev,noexec,
│ ├─/proc/sys/fs/binfmt_misc systemd-1 autofs rw,relatime,fd=32,pgrp=
│ └─/proc/fs/nfsd nfsd nfsd rw,relatime
├─/dev devtmpfs devtmpf rw,nosuid,size=8175208k
│ ├─/dev/shm tmpfs tmpfs rw,nosuid,nodev
│ ├─/dev/pts devpts devpts rw,nosuid,noexec,relati
│ ├─/dev/mqueue mqueue mqueue rw,relatime
│ └─/dev/hugepages hugetlbfs hugetlb rw,relatime
├─/run tmpfs tmpfs rw,nosuid,nodev,mode=75
│ ├─/run/user/42 tmpfs tmpfs rw,nosuid,nodev,relatim
│ │ └─/run/user/42/gvfs gvfsd-fuse fuse.gv rw,nosuid,nodev,relatim
│ └─/run/user/1000 tmpfs tmpfs rw,nosuid,nodev,relatim
│ └─/run/user/1000/gvfs gvfsd-fuse fuse.gv rw,nosuid,nodev,relatim
├─/tmp tmpfs tmpfs rw
├─/boot /dev/sda2 ext4 rw,relatime,data=ordere
│ └─/boot/efi /dev/sda1 vfat rw,relatime,fmask=0077,
├─/var/lib/nfs/rpc_pipefs sunrpc rpc_pip rw,relatime
├─/var/lib/kubelet/pods/fd20f710-fb82-11e5-ab9f-0862662cf845/volumes/kubernetes.io~secret/default-token-qggyv
│ tmpfs tmpfs rw,relatime
├─/var/lib/kubelet/pods/2f652e15-fb83-11e5-ab9f-0862662cf845/volumes/kubernetes.io~configmap/config-volume
│ tmpfs tmpfs rw,relatime
└─/var/lib/kubelet/pods/2f652e15-fb83-11e5-ab9f-0862662cf845/volumes/kubernetes.io~secret/default-token-6bzfe
tmpfs tmpfs rw,relatime
[schou#totoro single-node]$
Thanks to #Paul Morie for helping me diagnose and fix this (from github issue):
bingo, the mount propagation mode of /var/lib/kubelet is private. try changing the mount flag for the kubelet dir to -v /var/lib/kubelet:/var/lib/kubelet:rw,shared
I also had to change MountFlags=slave to MountFlags=shared in my docker systemd file.