Running two containers in a single pod via kubectl run does not terminate the pod with two containers - mongodb

Question

Trying to run a two docker containers in one kubernetes pod. The problem we face is that we have 2 images/containers in a Kubernetes pod whereas we used to have 1 image for this pod previously where it was working perfectly and shutting down the pod gracefully. Now the problem is that with 2 images or containers within the same pod both images are not shutting down at the same time gracefully.
We are doing integration testing of our container against the database which is MongoDB, our container terminates but MongoDB fails to terminate here and continues running
while getopts n:c:l:i:d:t: option
do
case "${option}"
in
n) Company_NAMESPACE=$OPTARG;;
c) Company_CONFIG=$OPTARG;;
l) Company_LICENSE=$OPTARG;;
i) Company_IMAGE=$OPTARG;;
d) Company_CONTAINER_NAME=$OPTARG;;
t) Company_TOKEN=$OPTARG;;
esac
done
kubectl run $Company_CONTAINER_NAME -n $Company_NAMESPACE --restart=Never --overrides='
{
"apiVersion": "v1",
"spec": {
"imagePullSecrets": [
{
"name": "Company-regsecret"
}
],
"initContainers": [
{
"name": "copy-configs",
"image": "busybox",
"command": ["sh", "-c", "cp /tmp/Company-config-volume/server/* /tmp/ng-rt/config/server/ 2>/dev/null || true; cp /tmp/Company-license-volume/licenses/* /tmp/ng-rt/config/licenses 2>/dev/null || true"],
"volumeMounts": [
{
"name": "Company-config-volume",
"mountPath": "mount_path"
},
{
"name": "'$Company_CONFIG'",
"mountPath": "mount_path"
},
{
"name": "Company-license-volume",
"mountPath": "mount_path"
},
{
"name": "'$Company_LICENSE'",
"mountPath": "mount_path"
}
]
}
],
"containers": [
{
"name": "mongodb-test",
"image": "mongo:3.6.8",
"command": [
"numactl",
"--interleave=all",
"mongod",
"--wiredTigerCacheSizeGB",
"0.1",
"--replSet",
"MainRepSet",
"--bind_ip_all"
],
"ports": [{
"containerPort": 27017
}],
"readinessProbe": {
"exec": {
"command": ["mongo", "--eval", "rs.initiate()"]
}
},
"terminationGracePeriodSeconds": 10
},
{
"env": [
{
"name": "AWS_ACCESS_KEY_ID",
"valueFrom": {
"secretKeyRef": {
"key": "AWS_ACCESS_KEY_ID",
"name": "aws-secrets"
}
}
},
{
"name": "AWS_SECRET_ACCESS_KEY",
"valueFrom": {
"secretKeyRef": {
"key": "AWS_SECRET_ACCESS_KEY",
"name": "aws-secrets"
}
}
},
{
"name": "AWS_REGION",
"valueFrom": {
"secretKeyRef": {
"key": "AWS_REGION",
"name": "aws-secrets"
}
}
},
{
"name": "BUILD_ID",
"valueFrom": {
"configMapKeyRef": {
"key": "BUILD_ID",
"name": "config"
}
}
}
],
"command": [
"sh",
"-c",
"mkdir -p mount_path 2\u003e/dev/null && npm test --skipConnectivityTestRethinkDB"
],
"name": "'$Company_CONTAINER_NAME'",
"image": "'$Company_IMAGE'",
"volumeMounts": [
{
"mountPath": "mount_path",
"name": "'$Company_CONFIG'"
},
{
"mountPath": "mount_path",
"name": "'$Company_LICENSE'"
}
]
}
],
"volumes": [
{
"name": "Company-config-volume",
"configMap": {
"name": "'$Company_CONFIG'"
}
},
{
"name": "'$Company_CONFIG'",
"emptyDir": {}
},
{
"name": "Company-license-volume",
"configMap": {
"name": "'$Company_LICENSE'"
}
},
{
"name": "'$Company_LICENSE'",
"emptyDir": {}
}
]
}
}
' --image=$Company_IMAGE -ti --rm --token=$Company_TOKEN
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "test-dev-58663",
"namespace": "d-int-company-dev-v2",
"labels": {
"run": "test-dev-58663"
},
"annotations": {
"kubernetes.io/psp": "nfs-provisioner"
}
},
"spec": {
"volumes": [
{
"name": "company-config-volume",
"configMap": {
"name": "test-core",
"defaultMode": 420
}
},
{
"name": "test-core",
"emptyDir": {}
},
{
"name": "company-license-volume",
"configMap": {
"name": "company-license",
"defaultMode": 420
}
},
{
"name": "company-license",
"emptyDir": {}
},
{
"name": "default-token-wqp5x",
"secret": {
"secretName": "default-token-wqp5x",
"defaultMode": 420
}
}
],
"initContainers": [
{
"name": "copy-configs",
"image": "busybox",
"command": [
"sh",
"-c",
"cp mount_path* mount_path 2>/dev/null || true; cp mount_path* mount_path 2>mount_path|| true"
],
"resources": {},
"volumeMounts": [
{
"name": "company-config-volume",
"mountPath": "mount_path"
},
{
"name": "test-core",
"mountPath": "mount_path"
},
{
"name": "company-license-volume",
"mountPath": "mount_path"
},
{
"name": "company-license",
"mountPath": "mount_path"
},
{
"name": "default-token-wqp5x",
"readOnly": true,
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "Always"
}
],
"containers": [
{
"name": "mongodb-test",
"image": "mongo:3.6.8",
"ports": [
{
"containerPort": 27017,
"protocol": "TCP"
}
],
"resources": {},
"volumeMounts": [
{
"name": "default-token-wqp5x",
"readOnly": true,
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
}
],
"readinessProbe": {
"exec": {
"command": [
"mongo",
"--eval",
"rs.initiate()"
]
},
"timeoutSeconds": 1,
"periodSeconds": 10,
"successThreshold": 1,
"failureThreshold": 3
},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent"
},
{
"name": "test-dev-58663",
"image": "image_path",
"command": [
"sh",
"-c",
"mkdir -p mount_path 2>/dev/null && npm test --skipConnectivityTestRethinkDB"
],
"env": [
{
"name": "AWS_ACCESS_KEY_ID",
"valueFrom": {
"secretKeyRef": {
"name": "aws-secrets",
"key": "AWS_ACCESS_KEY_ID"
}
}
},
{
"name": "AWS_SECRET_ACCESS_KEY",
"valueFrom": {
"secretKeyRef": {
"name": "aws-secrets",
"key": "AWS_SECRET_ACCESS_KEY"
}
}
},
{
"name": "AWS_REGION",
"valueFrom": {
"secretKeyRef": {
"name": "aws-secrets",
"key": "AWS_REGION"
}
}
},
{
"name": "BUILD_ID",
"valueFrom": {
"configMapKeyRef": {
"name": "tbsp-config",
"key": "BUILD_ID"
}
}
}
],
"resources": {},
"volumeMounts": [
{
"name": "test-core",
"mountPath": "mount_path"
},
{
"name": "company-license",
"mountPath": "mount_path"
},
{
"name": "default-token-wqp5x",
"readOnly": true,
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent"
}
],
"restartPolicy": "Never",
"terminationGracePeriodSeconds": 10,
"dnsPolicy": "ClusterFirst",
"serviceAccountName": "default",
"serviceAccount": "default",
"nodeName": "pc212",
"securityContext": {},
"imagePullSecrets": [
{
"name": "company-regsecret"
}
],
"schedulerName": "default-scheduler",
"tolerations": [
{
"key": "node.kubernetes.io/not-ready",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": 300
},
{
"key": "node.kubernetes.io/unreachable",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": 300
}
]
},
"status": {
"phase": "Pending",
"conditions": [
{
"type": "Initialized",
"status": "True",
"lastProbeTime": null,
"lastTransitionTime": "2019-05-10T12:50:39Z"
},
{
"type": "Ready",
"status": "False",
"lastProbeTime": null,
"lastTransitionTime": "2019-05-10T12:49:54Z",
"reason": "ContainersNotReady",
"message": "containers with unready status: [mongodb-test test-dev-58663]"
},
{
"type": "PodScheduled",
"status": "True",
"lastProbeTime": null,
"lastTransitionTime": "2019-05-10T12:49:54Z"
}
],
"hostIP": "10.10.2.12",
"podIP": "172.16.4.22",
"startTime": "2019-05-10T12:49:54Z",
"initContainerStatuses": [
{
"name": "copy-configs",
"state": {
"terminated": {
"exitCode": 0,
"reason": "Completed",
"startedAt": "2019-05-10T12:50:39Z",
"finishedAt": "2019-05-10T12:50:39Z",
"containerID": "docker://1bcd12f5848e32e82f7dfde8e245223345e87f70061b789cbbabc0f798436b59"
}
},
"lastState": {},
"ready": true,
"restartCount": 0,
"image": "busybox:latest",
"imageID": "docker-pullable://busybox#sha256:0b184b74edc63924be0d7f67d16f5afbcdbe61caa1aca9312ed3b5c57792f6c1",
"containerID": "docker://1bcd12f5848e32e82f7dfde8e245223345e87f70061b789cbbabc0f798436b59"
}
],
"containerStatuses": [
{
"name": "mongodb-test",
"state": {
"waiting": {
"reason": "PodInitializing"
}
},
"lastState": {},
"ready": false,
"restartCount": 0,
"image": "mongo:3.6.8",
"imageID": ""
},
{
"name": "test-dev-58663",
"state": {
"waiting": {
"reason": "PodInitializing"
}
},
"lastState": {},
"ready": false,
"restartCount": 0,
"image": "image_path",
"imageID": ""
}
],
"qosClass": "BestEffort"
}
}
Both the containers and the hosting pod should terminate gracefully.

Answer 1

You have pod consisting of two containers, one of them is supposed to run indefinitely, another one runs to completion. This is bad practice. You should split your pod into two separate things: Pod with Mongo and Job with your integration script. You have to write logic that watches Job to finish and then terminates both Pod and Job. You can do it like this:
kubectl apply -f integration-test.yaml
kubectl wait --for=condition=Complete --timeout=5m job/test
kubectl delete -f integration-test.yaml