Edit/Update a pod's yaml - kubernetes
I have a pod in my EKS cluster and I want to edit it's yaml so that I can change the read-only values from true to false . This way I want to be able to make changes to the pod's system/image (haven't exactly figured out it's name) that at the moment it read-only file system.
Is that possible ? Can I do that ?
I tried copying the current yaml contents and creating a new yaml file with the read-only values setted to false ,in order to use it as a replacement for the current one.
The command I tried to use is:
kubectl apply -f telegraf-new.yaml --namespace examplenamespace -l app=polling-telegraf-s
and the error I get is:
Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply
The Pod "polling-telegraf-s-79f44d578f-khdjf" is invalid: spec: Forbidden: pod updates may not change fields other than spec.containers[*].image, spec.initContainers[*].image, spec.activeDeadlineSeconds or spec.tolerations (only additions to existing tolerations)
I am not sure that this is a good way to approach my problem but I spend the last few days researching about it and the results are not so encouraging.
Any help,tip,advice to the correct direction would be appreciated.
Edit:
My yaml from the kubectl get pod --namespace tick -l app=polling-telegraf-s -o yaml is :
apiVersion: v1
items:
- apiVersion: v1
kind: Pod
metadata:
annotations:
checksum/config: 45cc44098254d90e88878e037f6eb5803be739890e26d9070e21ac0c0650debd
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"checksum/config":"45cc44098254d90e88878e037f6eb5803be739890e26d9070e21ac0c0650debd","kubernetes.io/psp":"eks.privileged"},"creationTimestamp":"2019-10-30T15:49:57Z","generateName":"polling-telegraf-s-79f44d578f-","labels":{"app":"polling-telegraf-s","pod-template-hash":"79f44d578f"},"name":"polling-telegraf-s-79f44d578f-khdjf","namespace":"tick","ownerReferences":[{"apiVersion":"apps/v1","blockOwnerDeletion":true,"controller":true,"kind":"ReplicaSet","name":"polling-telegraf-s-79f44d578f","uid":"ec1e6988-fb2c-11e9-bdf2-02b7fbdf557a"}],"resourceVersion":"134887","selfLink":"/api/v1/namespaces/tick/pods/polling-telegraf-s-79f44d578f-khdjf","uid":"ec1fa8a5-fb2c-11e9-bdf2-02b7fbdf557a"},"spec":{"containers":[{"image":"telegraf:1.10.3-alpine","imagePullPolicy":"IfNotPresent","name":"polling-telegraf-s","resources":{"limits":{"cpu":"1","memory":"2Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/etc/telegraf","name":"config"},{"mountPath":"/var/run/utmp","name":"varrunutmpro","readOnly":true},{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount","name":"default-token-htxsr","readOnly":true}]}],"dnsPolicy":"ClusterFirst","enableServiceLinks":true,"nodeName":"ip-192-168-179-5.eu-west-2.compute.internal","priority":0,"restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"serviceAccount":"default","serviceAccountName":"default","terminationGracePeriodSeconds":30,"tolerations":[{"effect":"NoExecute","key":"node.kubernetes.io/not-ready","operator":"Exists","tolerationSeconds":300},{"effect":"NoExecute","key":"node.kubernetes.io/unreachable","operator":"Exists","tolerationSeconds":300}],"volumes":[{"hostPath":{"path":"/var/run/utmp","type":""},"name":"varrunutmpro"},{"configMap":{"defaultMode":420,"name":"polling-telegraf-s"},"name":"config"},{"name":"default-token-htxsr","secret":{"defaultMode":420,"secretName":"default-token-htxsr"}}]},"status":{"conditions":[{"lastProbeTime":null,"lastTransitionTime":"2019-10-30T15:49:57Z","status":"True","type":"Initialized"},{"lastProbeTime":null,"lastTransitionTime":"2019-10-30T15:49:58Z","status":"True","type":"Ready"},{"lastProbeTime":null,"lastTransitionTime":"2019-10-30T15:49:58Z","status":"True","type":"ContainersReady"},{"lastProbeTime":null,"lastTransitionTime":"2019-10-30T15:49:57Z","status":"True","type":"PodScheduled"}],"containerStatuses":[{"containerID":"docker://a66f40111474ea28d1b1b7adf6d9e0278adb6d6aefa23b345cc1559174018f27","image":"telegraf:1.10.3-alpine","imageID":"docker-pullable://telegraf#sha256:9106295bc67459633b4d6151c2e1b9949e501560b2e659fe541bda691c566bcf","lastState":{},"name":"polling-telegraf-s","ready":true,"restartCount":0,"state":{"running":{"startedAt":"2019-10-30T15:49:58Z"}}}],"hostIP":"192.168.179.5","phase":"Running","podIP":"192.168.159.179","qosClass":"Burstable","startTime":"2019-10-30T15:49:57Z"}}
kubernetes.io/psp: eks.privileged
creationTimestamp: "2019-10-30T15:49:57Z"
generateName: polling-telegraf-s-79f44d578f-
labels:
app: polling-telegraf-s
pod-template-hash: 79f44d578f
name: polling-telegraf-s-79f44d578f-khdjf
namespace: tick
ownerReferences:
- apiVersion: apps/v1
blockOwnerDeletion: true
controller: true
kind: ReplicaSet
name: polling-telegraf-s-79f44d578f
uid: ec1e6988-fb2c-11e9-bdf2-02b7fbdf557a
resourceVersion: "409255"
selfLink: /api/v1/namespaces/tick/pods/polling-telegraf-s-79f44d578f-khdjf
uid: ec1fa8a5-fb2c-11e9-bdf2-02b7fbdf557a
spec:
containers:
- image: telegraf:1.10.3-alpine
imagePullPolicy: IfNotPresent
name: polling-telegraf-s
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: 100m
memory: 256Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/telegraf
name: config
- mountPath: /var/run/utmp
name: varrunutmpro
readOnly: true
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: default-token-htxsr
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
nodeName: ip-192-168-179-5.eu-west-2.compute.internal
priority: 0
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: default
serviceAccountName: default
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
volumes:
- hostPath:
path: /var/run/utmp
type: ""
name: varrunutmpro
- configMap:
defaultMode: 420
name: polling-telegraf-s
name: config
- name: default-token-htxsr
secret:
defaultMode: 420
secretName: default-token-htxsr
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2019-10-30T15:49:57Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2019-10-30T15:49:58Z"
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2019-10-30T15:49:58Z"
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2019-10-30T15:49:57Z"
status: "True"
type: PodScheduled
containerStatuses:
- containerID: docker://a66f40111474ea28d1b1b7adf6d9e0278adb6d6aefa23b345cc1559174018f27
image: telegraf:1.10.3-alpine
imageID: docker-pullable://telegraf#sha256:9106295bc67459633b4d6151c2e1b9949e501560b2e659fe541bda691c566bcf
lastState: {}
name: polling-telegraf-s
ready: true
restartCount: 0
state:
running:
startedAt: "2019-10-30T15:49:58Z"
hostIP: 192.168.179.5
phase: Running
podIP: 192.168.159.179
qosClass: Burstable
startTime: "2019-10-30T15:49:57Z"
kind: List
metadata:
resourceVersion: ""
selfLink: ""
and I want to change the readOnly values from true to false .
You can edit a kubernetes resource yaml using the command kubectl edit [resource] [UID]. For instance, to change the yaml of a pod, you would run kubectl edit $POD_UID
However this won't work in your case because you are editing a mount and that requires the pod to be restarted. In this sense, the best approach would be what you already did. Start by extracting the yaml from the api by running kubectl get pod $POD_UID -o yaml and then editing its content to deploy it again. However as the error depicts, you are not allowed to edit some parts of the yaml. So you have to remove those parts.
Only keep the parts that the log has already pointed to: spec.containers[*].image, spec.initContainers[*].image, spec.activeDeadlineSeconds or spec.tolerations
Otherwise, here are the sections you would have to remove:
metadata:
creationTimestamp: 2019-11-01T13:22:50Z
generateName:
ownerReferences:
- apiVersion: apps/v1
blockOwnerDeletion: true
controller: true
kind: ReplicaSet
name:
uid: 947fb7b7-f1ab-11e9-adfb-42010a8001b2
resourceVersion: "103002009"
selfLink:
uid: b3f96ba4-fcaa-11e9-adfb-42010a8001b2
spec:
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: default-token-ff27n
readOnly: true
nodeName:
priority: 0
schedulerName: default-scheduler
securityContext: {}
serviceAccount: default
serviceAccountName: default
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
volumes:
- name: default-token-ff27n
secret:
defaultMode: 420
secretName: default-token-ff27n
status:
conditions:
- lastProbeTime: null
lastTransitionTime: 2019-11-01T13:22:50Z
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: 2019-11-01T13:22:55Z
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: null
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: 2019-11-01T13:22:50Z
status: "True"
type: PodScheduled
containerStatuses:
- containerID:
image:
imageID:
lastState: {}
name: proxy
ready: true
restartCount: 0
state:
running:
startedAt: 2019-11-01T13:22:55Z
hostIP:
phase: Running
podIP:
qosClass: Burstable
startTime: 2019-11-01T13:22:50Z
You can edit pod yaml on the fly using kubectl edit pods <pod-name>.
You have to keep in mind that there are fields which will not be allowed to be edited while pod is scheduled, this is mentioned in your error message.
I think you should first remove the pod and apply the new yaml file.
Related
Prometheus & Alert Manager keeps crashing after updating the EKS version to 1.16
prometheus-prometheus-kube-prometheus-prometheus-0 0/2 Terminating 0 4s alertmanager-prometheus-kube-prometheus-alertmanager-0 0/2 Terminating 0 10s After updating EKS cluster to 1.16 from 1.15 everything works fine except these two pods, they keep on terminating and unable to initialise. Hence, prometheus monitoring does not work. I am getting below errors while describing the pods. Error: failed to start container "prometheus": Error response from daemon: OCI runtime create failed: container_linux.go:362: creating new parent process caused: container_linux.go:1941: running lstat on namespace path "/proc/29271/ns/ipc" caused: lstat /proc/29271/ns/ipc: no such file or directory: unknown Error: failed to start container "config-reloader": Error response from daemon: cannot join network of a non running container: 7e139521980afd13dad0162d6859352b0b2c855773d6d4062ee3e2f7f822a0b3 Error: cannot find volume "config" to mount into container "config-reloader" Error: cannot find volume "config" to mount into container "prometheus" here is my yaml file for the deployment: apiVersion: v1 kind: Pod metadata: annotations: kubernetes.io/psp: eks.privileged creationTimestamp: "2021-04-30T16:39:14Z" deletionGracePeriodSeconds: 600 deletionTimestamp: "2021-04-30T16:49:14Z" generateName: prometheus-prometheus-kube-prometheus-prometheus- labels: app: prometheus app.kubernetes.io/instance: prometheus-kube-prometheus-prometheus app.kubernetes.io/managed-by: prometheus-operator app.kubernetes.io/name: prometheus app.kubernetes.io/version: 2.26.0 controller-revision-hash: prometheus-prometheus-kube-prometheus-prometheus-56d9fcf57 operator.prometheus.io/name: prometheus-kube-prometheus-prometheus operator.prometheus.io/shard: "0" prometheus: prometheus-kube-prometheus-prometheus statefulset.kubernetes.io/pod-name: prometheus-prometheus-kube-prometheus-prometheus-0 name: prometheus-prometheus-kube-prometheus-prometheus-0 namespace: mo ownerReferences: - apiVersion: apps/v1 blockOwnerDeletion: true controller: true kind: StatefulSet name: prometheus-prometheus-kube-prometheus-prometheus uid: 326a09f2-319c-449d-904a-1dd0019c6d80 resourceVersion: "9337443" selfLink: /api/v1/namespaces/monitoring/pods/prometheus-prometheus-kube-prometheus-prometheus-0 uid: e2be062f-749d-488e-a6cc-42ef1396851b spec: containers: - args: - --web.console.templates=/etc/prometheus/consoles - --web.console.libraries=/etc/prometheus/console_libraries - --config.file=/etc/prometheus/config_out/prometheus.env.yaml - --storage.tsdb.path=/prometheus - --storage.tsdb.retention.time=10d - --web.enable-lifecycle - --storage.tsdb.no-lockfile - --web.external-url=http://prometheus-kube-prometheus-prometheus.monitoring:9090 - --web.route-prefix=/ image: quay.io/prometheus/prometheus:v2.26.0 imagePullPolicy: IfNotPresent name: prometheus ports: - containerPort: 9090 name: web protocol: TCP readinessProbe: failureThreshold: 120 httpGet: path: /-/ready port: web scheme: HTTP periodSeconds: 5 successThreshold: 1 timeoutSeconds: 3 resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/prometheus/config_out name: config-out readOnly: true - mountPath: /etc/prometheus/certs name: tls-assets readOnly: true - mountPath: /prometheus name: prometheus-prometheus-kube-prometheus-prometheus-db - mountPath: /etc/prometheus/rules/prometheus-prometheus-kube-prometheus-prometheus-rulefiles-0 name: prometheus-prometheus-kube-prometheus-prometheus-rulefiles-0 - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: prometheus-kube-prometheus-prometheus-token-mh66q readOnly: true - args: - --listen-address=:8080 - --reload-url=http://localhost:9090/-/reload - --config-file=/etc/prometheus/config/prometheus.yaml.gz - --config-envsubst-file=/etc/prometheus/config_out/prometheus.env.yaml - --watched-dir=/etc/prometheus/rules/prometheus-prometheus-kube-prometheus-prometheus-rulefiles-0 command: - /bin/prometheus-config-reloader env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: SHARD value: "0" image: quay.io/prometheus-operator/prometheus-config-reloader:v0.47.0 imagePullPolicy: IfNotPresent name: config-reloader ports: - containerPort: 8080 name: reloader-web protocol: TCP resources: limits: cpu: 100m memory: 50Mi requests: cpu: 100m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/prometheus/config name: config - mountPath: /etc/prometheus/config_out name: config-out - mountPath: /etc/prometheus/rules/prometheus-prometheus-kube-prometheus-prometheus-rulefiles-0 name: prometheus-prometheus-kube-prometheus-prometheus-rulefiles-0 - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: prometheus-kube-prometheus-prometheus-token-mh66q readOnly: true dnsPolicy: ClusterFirst enableServiceLinks: true hostname: prometheus-prometheus-kube-prometheus-prometheus-0 nodeName: ip-10-1-49-45.ec2.internal priority: 0 restartPolicy: Always schedulerName: default-scheduler securityContext: fsGroup: 2000 runAsGroup: 2000 runAsNonRoot: true runAsUser: 1000 serviceAccount: prometheus-kube-prometheus-prometheus serviceAccountName: prometheus-kube-prometheus-prometheus subdomain: prometheus-operated terminationGracePeriodSeconds: 600 tolerations: - effect: NoExecute key: node.kubernetes.io/not-ready operator: Exists tolerationSeconds: 300 - effect: NoExecute key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 300 volumes: - name: config secret: defaultMode: 420 secretName: prometheus-prometheus-kube-prometheus-prometheus - name: tls-assets secret: defaultMode: 420 secretName: prometheus-prometheus-kube-prometheus-prometheus-tls-assets - emptyDir: {} name: config-out - configMap: defaultMode: 420 name: prometheus-prometheus-kube-prometheus-prometheus-rulefiles-0 name: prometheus-prometheus-kube-prometheus-prometheus-rulefiles-0 - emptyDir: {} name: prometheus-prometheus-kube-prometheus-prometheus-db - name: prometheus-kube-prometheus-prometheus-token-mh66q secret: defaultMode: 420 secretName: prometheus-kube-prometheus-prometheus-token-mh66q status: conditions: - lastProbeTime: null lastTransitionTime: "2021-04-30T16:39:14Z" status: "True" type: PodScheduled phase: Pending qosClass: Burstable
If someone needs to know the answer, in my case(the above situation) there were 2 Prometheus operators running in different different namespace, 1 in default & another monitoring namespace. so I removed the one from the default namespace and it resolved my pods crashing issue.
Unable to deploy mongodb community operator in openshift
I'm trying to deploy the mongodb community operator in openshift 3.11, using the following commands: git clone https://github.com/mongodb/mongodb-kubernetes-operator.git cd mongodb-kubernetes-operator oc new-project mongodb oc create -f deploy/crds/mongodb.com_mongodb_crd.yaml -n mongodb oc create -f deploy/operator/role.yaml -n mongodb oc create -f deploy/operator/role_binding.yaml -n mongodb oc create -f deploy/operator/service_account.yaml -n mongodb oc apply -f deploy/openshift/operator_openshift.yaml -n mongodb oc apply -f deploy/crds/mongodb.com_v1_mongodb_openshift_cr.yaml -n mongodb Operator pod is successfully running, but the mongodb replicaset pods do not spin up. Error is as follows: [kubenode#master mongodb-kubernetes-operator]$ oc get pods NAME READY STATUS RESTARTS AGE example-openshift-mongodb-0 1/2 CrashLoopBackOff 4 2m mongodb-kubernetes-operator-66bfcbcf44-9xvj7 1/1 Running 0 2m [kubenode#master mongodb-kubernetes-operator]$ oc logs -f example-openshift-mongodb-0 -c mongodb-agent panic: Failed to get current user: user: unknown userid 1000510000 goroutine 1 [running]: com.tengen/cm/util.init.3() /data/mci/2f46ec94982c5440960d2b2bf2b6ae15/mms-automation/build/go-dependencies/src/com.tengen/cm/util/user.go:14 +0xe5 I have gone through all the issues raised on the mongodb-kubernetes-operator repository which are related to this issue (reference), and found a suggestion to set the MANAGED_SECURITY_CONTEXT environment variable to true in the operator, mongodb and mongodb-agent containers. I have done so for all of these containers, but am still facing the same issue. Here is the confirmation that the environment variables are correctly set: [kubenode#master mongodb-kubernetes-operator]$ oc set env statefulset.apps/example-openshift-mongodb --list # statefulsets/example-openshift-mongodb, container mongodb-agent AGENT_STATUS_FILEPATH=/var/log/mongodb-mms-automation/healthstatus/agent-health-status.json AUTOMATION_CONFIG_MAP=example-openshift-mongodb-config HEADLESS_AGENT=true MANAGED_SECURITY_CONTEXT=true # POD_NAMESPACE from field path metadata.namespace # statefulsets/example-openshift-mongodb, container mongod AGENT_STATUS_FILEPATH=/healthstatus/agent-health-status.json MANAGED_SECURITY_CONTEXT=true [kubenode#master mongodb-kubernetes-operator]$ oc set env deployment.apps/mongodb-kubernetes-operator --list # deployments/mongodb-kubernetes-operator, container mongodb-kubernetes-operator # WATCH_NAMESPACE from field path metadata.namespace # POD_NAME from field path metadata.name MANAGED_SECURITY_CONTEXT=true OPERATOR_NAME=mongodb-kubernetes-operator AGENT_IMAGE=quay.io/mongodb/mongodb-agent:10.19.0.6562-1 VERSION_UPGRADE_HOOK_IMAGE=quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 Operator Information Operator Version: 0.3.0 MongoDB Image used: 4.2.6 Cluster Information [kubenode#master mongodb-kubernetes-operator]$ openshift version openshift v3.11.0+62803d0-1 [kubenode#master mongodb-kubernetes-operator]$ kubectl version Client Version: version.Info{Major:"1", Minor:"11+", GitVersion:"v1.11.0+d4cacc0", GitCommit:"d4cacc0", GitTreeState:"clean", BuildDate:"2018-10-15T09:45:30Z", GoVersion:"go1.10.2", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"11+", GitVersion:"v1.11.0+d4cacc0", GitCommit:"d4cacc0", GitTreeState:"clean", BuildDate:"2020-12-07T17:59:40Z", GoVersion:"go1.10.8", Compiler:"gc", Platform:"linux/amd64"} Update When I check the replica pod yaml (see below), I see three occurrences of runAsUser security context set as 1000510000. I'm not sure how, but this is being set even though I'm not setting it manually. [kubenode#master mongodb-kubernetes-operator]$ oc get -o yaml pod example-openshift-mongodb-0 apiVersion: v1 kind: Pod metadata: annotations: openshift.io/scc: restricted creationTimestamp: 2021-01-19T07:45:05Z generateName: example-openshift-mongodb- labels: app: example-openshift-mongodb-svc controller-revision-hash: example-openshift-mongodb-6549495b statefulset.kubernetes.io/pod-name: example-openshift-mongodb-0 name: example-openshift-mongodb-0 namespace: mongodb ownerReferences: - apiVersion: apps/v1 blockOwnerDeletion: true controller: true kind: StatefulSet name: example-openshift-mongodb uid: 3e91eb40-5a2a-11eb-a5e0-0050569b1f59 resourceVersion: "15616863" selfLink: /api/v1/namespaces/mongodb/pods/example-openshift-mongodb-0 uid: 3ea17a28-5a2a-11eb-a5e0-0050569b1f59 spec: containers: - command: - agent/mongodb-agent - -cluster=/var/lib/automation/config/cluster-config.json - -skipMongoStart - -noDaemonize - -healthCheckFilePath=/var/log/mongodb-mms-automation/healthstatus/agent-health-status.json - -serveStatusPort=5000 - -useLocalMongoDbTools env: - name: AGENT_STATUS_FILEPATH value: /var/log/mongodb-mms-automation/healthstatus/agent-health-status.json - name: AUTOMATION_CONFIG_MAP value: example-openshift-mongodb-config - name: HEADLESS_AGENT value: "true" - name: MANAGED_SECURITY_CONTEXT value: "true" - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace image: quay.io/mongodb/mongodb-agent:10.19.0.6562-1 imagePullPolicy: Always name: mongodb-agent readinessProbe: exec: command: - /var/lib/mongodb-mms-automation/probes/readinessprobe failureThreshold: 60 initialDelaySeconds: 5 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 resources: {} securityContext: capabilities: drop: - KILL - MKNOD - SETGID - SETUID runAsUser: 1000510000 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /var/lib/automation/config name: automation-config readOnly: true - mountPath: /data name: data-volume - mountPath: /var/lib/mongodb-mms-automation/authentication name: example-openshift-mongodb-agent-scram-credentials - mountPath: /var/log/mongodb-mms-automation/healthstatus name: healthstatus - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: mongodb-kubernetes-operator-token-lr9l4 readOnly: true - command: - /bin/sh - -c - |2 # run post-start hook to handle version changes /hooks/version-upgrade # wait for config to be created by the agent while [ ! -f /data/automation-mongod.conf ]; do sleep 3 ; done ; sleep 2 ; # start mongod with this configuration exec mongod -f /data/automation-mongod.conf ; env: - name: AGENT_STATUS_FILEPATH value: /healthstatus/agent-health-status.json - name: MANAGED_SECURITY_CONTEXT value: "true" image: mongo:4.2.6 imagePullPolicy: IfNotPresent name: mongod resources: {} securityContext: capabilities: drop: - KILL - MKNOD - SETGID - SETUID runAsUser: 1000510000 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /data name: data-volume - mountPath: /var/lib/mongodb-mms-automation/authentication name: example-openshift-mongodb-agent-scram-credentials - mountPath: /healthstatus name: healthstatus - mountPath: /hooks name: hooks - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: mongodb-kubernetes-operator-token-lr9l4 readOnly: true dnsPolicy: ClusterFirst hostname: example-openshift-mongodb-0 imagePullSecrets: - name: mongodb-kubernetes-operator-dockercfg-jhplw initContainers: - command: - cp - version-upgrade-hook - /hooks/version-upgrade image: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 imagePullPolicy: Always name: mongod-posthook resources: {} securityContext: capabilities: drop: - KILL - MKNOD - SETGID - SETUID runAsUser: 1000510000 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /hooks name: hooks - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: mongodb-kubernetes-operator-token-lr9l4 readOnly: true nodeName: node1.192.168.27.116.nip.io nodeSelector: node-role.kubernetes.io/compute: "true" priority: 0 restartPolicy: Always schedulerName: default-scheduler securityContext: fsGroup: 1000510000 seLinuxOptions: level: s0:c23,c2 serviceAccount: mongodb-kubernetes-operator serviceAccountName: mongodb-kubernetes-operator subdomain: example-openshift-mongodb-svc terminationGracePeriodSeconds: 30 volumes: - name: data-volume persistentVolumeClaim: claimName: data-volume-example-openshift-mongodb-0 - name: automation-config secret: defaultMode: 416 secretName: example-openshift-mongodb-config - name: example-openshift-mongodb-agent-scram-credentials secret: defaultMode: 384 secretName: example-openshift-mongodb-agent-scram-credentials - emptyDir: {} name: healthstatus - emptyDir: {} name: hooks - name: mongodb-kubernetes-operator-token-lr9l4 secret: defaultMode: 420 secretName: mongodb-kubernetes-operator-token-lr9l4 status: conditions: - lastProbeTime: null lastTransitionTime: 2021-01-19T07:46:45Z status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: 2021-01-19T07:46:39Z message: 'containers with unready status: [mongodb-agent]' reason: ContainersNotReady status: "False" type: Ready - lastProbeTime: null lastTransitionTime: null message: 'containers with unready status: [mongodb-agent]' reason: ContainersNotReady status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: 2021-01-19T07:45:05Z status: "True" type: PodScheduled containerStatuses: - containerID: docker://bd3ede9178bb78267bc19d1b5da0915d3bcd1d4dcee3e142c7583424bd2aa777 image: docker.io/mongo:4.2.6 imageID: docker-pullable://docker.io/mongo#sha256:c880f6b56f443bb4d01baa759883228cd84fa8d78fa1a36001d1c0a0712b5a07 lastState: {} name: mongod ready: true restartCount: 0 state: running: startedAt: 2021-01-19T07:46:55Z - containerID: docker://5e39c0b6269b8231bbf9cabb4ff3457d9f91e878eff23953e318a9475fb8a90e image: quay.io/mongodb/mongodb-agent:10.19.0.6562-1 imageID: docker-pullable://quay.io/mongodb/mongodb-agent#sha256:790c2670ef7cefd61cfaabaf739de16dbd2e07dc3b539add0da21ab7d5ac7626 lastState: terminated: containerID: docker://5e39c0b6269b8231bbf9cabb4ff3457d9f91e878eff23953e318a9475fb8a90e exitCode: 2 finishedAt: 2021-01-19T19:39:58Z reason: Error startedAt: 2021-01-19T19:39:58Z name: mongodb-agent ready: false restartCount: 144 state: waiting: message: Back-off 5m0s restarting failed container=mongodb-agent pod=example-openshift-mongodb-0_mongodb(3ea17a28-5a2a-11eb-a5e0-0050569b1f59) reason: CrashLoopBackOff hostIP: 192.168.27.116 initContainerStatuses: - containerID: docker://7c31cef2a68e3e6100c2cc9c83e3780313f1e8ab43bebca79ad4d48613f124bd image: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 imageID: docker-pullable://quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook#sha256:e99105b1c54e12913ddaf470af8025111a6e6e4c8917fc61be71d1bc0328e7d7 lastState: {} name: mongod-posthook ready: true restartCount: 0 state: terminated: containerID: docker://7c31cef2a68e3e6100c2cc9c83e3780313f1e8ab43bebca79ad4d48613f124bd exitCode: 0 finishedAt: 2021-01-19T07:46:45Z reason: Completed startedAt: 2021-01-19T07:46:44Z phase: Running podIP: 10.129.0.119 qosClass: BestEffort startTime: 2021-01-19T07:46:39Z
PriorityClass doesn't populate its value to podSpec
env: vagrant + virtualbox kubernetes: 1.14 docker 18.06.3~ce~3-0~debian os: debian stretch I have priority classes: root#k8s-master:/# kubectl get priorityclass NAME VALUE GLOBAL-DEFAULT AGE cluster-health-priority 1000000000 false 33m < -- created by me default-priority 100 true 33m < -- created by me system-cluster-critical 2000000000 false 33m < -- system system-node-critical 2000001000 false 33m < -- system default-priority - has been set as globalDefault root#k8s-master:/# kubectl get priorityclass default-priority -o yaml apiVersion: scheduling.k8s.io/v1 description: Used for all Pods without priorityClassName globalDefault: true <------------------ kind: PriorityClass metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"scheduling.k8s.io/v1","description":"Used for all Pods without priorityClassName","globalDefault":true,"kind":"PriorityClass","metadata":{"annotations":{},"labels":{"addonmanager.kubernetes.io/mode":"Reconcile"},"name":"default-priority"},"value":100} creationTimestamp: "2019-07-15T16:48:23Z" generation: 1 labels: addonmanager.kubernetes.io/mode: Reconcile name: default-priority resourceVersion: "304" selfLink: /apis/scheduling.k8s.io/v1/priorityclasses/default-priority uid: 5bea6f73-a720-11e9-8343-0800278dc04d value: 100 I have some pods, which were created after policy classes creation This kube-state-metrics-874ccb958-b5spd 1/1 Running 0 9m18s 10.20.59.67 k8s-master <none> <none> And this tmp-shell-one-59fb949cb5-b8khc 1/1 Running 1 47s 10.20.59.73 k8s-master <none> <none> kube-state-metrics pod is using priorityClass cluster-health-priority root#k8s-master:/etc/kubernetes/addons# kubectl -n kube-system get pod kube-state-metrics-874ccb958-b5spd -o yaml apiVersion: v1 kind: Pod metadata: creationTimestamp: "2019-07-15T16:48:24Z" generateName: kube-state-metrics-874ccb958- labels: k8s-app: kube-state-metrics pod-template-hash: 874ccb958 name: kube-state-metrics-874ccb958-b5spd namespace: kube-system ownerReferences: - apiVersion: apps/v1 blockOwnerDeletion: true controller: true kind: ReplicaSet name: kube-state-metrics-874ccb958 uid: 5c64bf85-a720-11e9-8343-0800278dc04d resourceVersion: "548" selfLink: /api/v1/namespaces/kube-system/pods/kube-state-metrics-874ccb958-b5spd uid: 5c88143e-a720-11e9-8343-0800278dc04d spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: kube-role operator: In values: - master containers: - image: gcr.io/google_containers/kube-state-metrics:v1.6.0 imagePullPolicy: Always name: kube-state-metrics ports: - containerPort: 8080 name: http-metrics protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /healthz port: 8080 scheme: HTTP initialDelaySeconds: 5 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-state-metrics-token-jvz5b readOnly: true dnsPolicy: ClusterFirst enableServiceLinks: true nodeName: k8s-master nodeSelector: namespaces/default: "true" priorityClassName: cluster-health-priority <------------------------ restartPolicy: Always schedulerName: default-scheduler securityContext: {} serviceAccount: kube-state-metrics serviceAccountName: kube-state-metrics terminationGracePeriodSeconds: 30 tolerations: - effect: NoSchedule key: dedicated operator: Equal value: master - key: CriticalAddonsOnly operator: Exists volumes: - name: kube-state-metrics-token-jvz5b secret: defaultMode: 420 secretName: kube-state-metrics-token-jvz5b status: conditions: - lastProbeTime: null lastTransitionTime: "2019-07-15T16:48:24Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2019-07-15T16:48:58Z" status: "True" type: Ready - lastProbeTime: null lastTransitionTime: "2019-07-15T16:48:58Z" status: "True" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2019-07-15T16:48:24Z" status: "True" type: PodScheduled containerStatuses: - containerID: docker://a736dce98492b7d746079728b683a2c62f6adb1068075ccc521c5e57ba1e02d1 image: gcr.io/google_containers/kube-state-metrics:v1.6.0 imageID: docker-pullable://gcr.io/google_containers/kube-state-metrics#sha256:c98991f50115fe6188d7b4213690628f0149cf160ac47daf9f21366d7cc62740 lastState: {} name: kube-state-metrics ready: true restartCount: 0 state: running: startedAt: "2019-07-15T16:48:46Z" hostIP: 10.0.2.15 phase: Running podIP: 10.20.59.67 qosClass: BestEffort startTime: "2019-07-15T16:48:24Z" tmp-shell pod has nothing about priority classes at all: apiVersion: v1 kind: Pod metadata: creationTimestamp: "2019-07-15T16:56:49Z" generateName: tmp-shell-one-59fb949cb5- labels: pod-template-hash: 59fb949cb5 run: tmp-shell-one name: tmp-shell-one-59fb949cb5-b8khc namespace: monitoring ownerReferences: - apiVersion: apps/v1 blockOwnerDeletion: true controller: true kind: ReplicaSet name: tmp-shell-one-59fb949cb5 uid: 89c3caa3-a721-11e9-8343-0800278dc04d resourceVersion: "1350" selfLink: /api/v1/namespaces/monitoring/pods/tmp-shell-one-59fb949cb5-b8khc uid: 89c71bad-a721-11e9-8343-0800278dc04d spec: containers: - args: - /bin/bash image: nicolaka/netshoot imagePullPolicy: Always name: tmp-shell-one resources: {} stdin: true terminationMessagePath: /dev/termination-log terminationMessagePolicy: File tty: true volumeMounts: - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: default-token-g9lnc readOnly: true dnsPolicy: ClusterFirst enableServiceLinks: true nodeName: k8s-master nodeSelector: namespaces/default: "true" restartPolicy: Always schedulerName: default-scheduler securityContext: {} serviceAccount: default serviceAccountName: default terminationGracePeriodSeconds: 30 volumes: - name: default-token-g9lnc secret: defaultMode: 420 secretName: default-token-g9lnc status: conditions: - lastProbeTime: null lastTransitionTime: "2019-07-15T16:56:49Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2019-07-15T16:57:20Z" status: "True" type: Ready - lastProbeTime: null lastTransitionTime: "2019-07-15T16:57:20Z" status: "True" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2019-07-15T16:56:49Z" status: "True" type: PodScheduled containerStatuses: - containerID: docker://545d4d029b440ebb694386abb09e0377164c87d1170ac79704f39d3167748bf5 image: nicolaka/netshoot:latest imageID: docker-pullable://nicolaka/netshoot#sha256:b3e662a8730ee51c6b877b6043c5b2fa61862e15d535e9f90cf667267407753f lastState: terminated: containerID: docker://dfdfd0d991151e94411029f2d5a1a81d67b5b55d43dcda017aec28320bafc7d3 exitCode: 130 finishedAt: "2019-07-15T16:57:17Z" reason: Error startedAt: "2019-07-15T16:57:03Z" name: tmp-shell-one ready: true restartCount: 1 state: running: startedAt: "2019-07-15T16:57:19Z" hostIP: 10.0.2.15 phase: Running podIP: 10.20.59.73 qosClass: BestEffort startTime: "2019-07-15T16:56:49Z" According to the docs: The globalDefault field indicates that the value of this PriorityClass should be used for Pods without a priorityClassName and Pod priority is specified by setting the priorityClassName field of podSpec. The integer value of priority is then resolved and populated to the priority field of podSpec So, the questions are: Why tmp-shell pod is not using priorityClass default-priority, even it created after priority class with globalDefault to true? Why kube-state-metrics pod does not have field priority with parsed value from the priority class cluster-health-priority in podSpec?(look at .yaml above) What am I doing wrong?
The only way I can reproduce it is by disabling the Priority Admission Controller by adding this argument --disable-admission-plugins=Priority to the kube-api-server definition which is under /etc/kubernetes/manifests/kube-apiserver.yaml of the Host running the API Server. According to the documentation in v1.14 this is enabled by default. Please make sure that it is enabled in your cluster as well.
Why Istio "Authentication Policy" Example Page isn't working as expected?
The article here: https://istio.io/docs/tasks/security/authn-policy/ Specifically, when I follow the instruction on the Setup section, I can't connect any httpbin that are residing in namespace foo and bar. But the legacy's one is okay. I expect there is something wrong in the side car proxy being installed. Here is the output of httpbin pod yaml file (after being injected with istioctl kubeinject --includeIPRanges "10.32.0.0/16" command). I use --includeIPRanges so that the pod can communicate with external ip (for my debugging purpose to install dnsutils, etc package) apiVersion: v1 kind: Pod metadata: annotations: sidecar.istio.io/inject: "true" sidecar.istio.io/status: '{"version":"4120ea817406fd7ed43b7ecf3f2e22abe453c44d3919389dcaff79b210c4cd86","initContainers":["istio-init"],"containers":["istio-proxy"],"volumes":["istio-envoy","istio-certs"],"imagePullSecrets":null}' creationTimestamp: 2018-08-15T11:40:59Z generateName: httpbin-8b9cf99f5- labels: app: httpbin pod-template-hash: "465795591" version: v1 name: httpbin-8b9cf99f5-9c47z namespace: foo ownerReferences: - apiVersion: extensions/v1beta1 blockOwnerDeletion: true controller: true kind: ReplicaSet name: httpbin-8b9cf99f5 uid: 1450d75d-a080-11e8-aece-42010a940168 resourceVersion: "65722138" selfLink: /api/v1/namespaces/foo/pods/httpbin-8b9cf99f5-9c47z uid: 1454b68d-a080-11e8-aece-42010a940168 spec: containers: - image: docker.io/citizenstig/httpbin imagePullPolicy: IfNotPresent name: httpbin ports: - containerPort: 8000 protocol: TCP resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: default-token-pkpvf readOnly: true - args: - proxy - sidecar - --configPath - /etc/istio/proxy - --binaryPath - /usr/local/bin/envoy - --serviceCluster - httpbin - --drainDuration - 45s - --parentShutdownDuration - 1m0s - --discoveryAddress - istio-pilot.istio-system:15007 - --discoveryRefreshDelay - 1s - --zipkinAddress - zipkin.istio-system:9411 - --connectTimeout - 10s - --statsdUdpAddress - istio-statsd-prom-bridge.istio-system.istio-system:9125 - --proxyAdminPort - "15000" - --controlPlaneAuthPolicy - NONE env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: INSTANCE_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.podIP - name: ISTIO_META_POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: ISTIO_META_INTERCEPTION_MODE value: REDIRECT image: docker.io/istio/proxyv2:1.0.0 imagePullPolicy: IfNotPresent name: istio-proxy resources: requests: cpu: 10m securityContext: privileged: false readOnlyRootFilesystem: true runAsUser: 1337 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /etc/istio/proxy name: istio-envoy - mountPath: /etc/certs/ name: istio-certs readOnly: true dnsPolicy: ClusterFirst initContainers: - args: - -p - "15001" - -u - "1337" - -m - REDIRECT - -i - 10.32.0.0/16 - -x - "" - -b - 8000, - -d - "" image: docker.io/istio/proxy_init:1.0.0 imagePullPolicy: IfNotPresent name: istio-init resources: {} securityContext: capabilities: add: - NET_ADMIN privileged: true terminationMessagePath: /dev/termination-log terminationMessagePolicy: File nodeName: gke-tvlk-data-dev-default-medium-pool-46397778-q2sb restartPolicy: Always schedulerName: default-scheduler securityContext: {} serviceAccount: default serviceAccountName: default terminationGracePeriodSeconds: 30 tolerations: - effect: NoExecute key: node.kubernetes.io/not-ready operator: Exists tolerationSeconds: 300 - effect: NoExecute key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 300 volumes: - name: default-token-pkpvf secret: defaultMode: 420 secretName: default-token-pkpvf - emptyDir: medium: Memory name: istio-envoy - name: istio-certs secret: defaultMode: 420 optional: true secretName: istio.default status: conditions: - lastProbeTime: null lastTransitionTime: 2018-08-15T11:41:01Z status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: 2018-08-15T11:44:28Z status: "True" type: Ready - lastProbeTime: null lastTransitionTime: 2018-08-15T11:40:59Z status: "True" type: PodScheduled containerStatuses: - containerID: docker://758e130a4c31a15c1b8bc1e1f72bd7739d5fa1103132861eea9ae1a6ae1f080e image: citizenstig/httpbin:latest imageID: docker-pullable://citizenstig/httpbin#sha256:b81c818ccb8668575eb3771de2f72f8a5530b515365842ad374db76ad8bcf875 lastState: {} name: httpbin ready: true restartCount: 0 state: running: startedAt: 2018-08-15T11:41:01Z - containerID: docker://9c78eac46a99457f628493975f5b0c5bbffa1dac96dab5521d2efe4143219575 image: istio/proxyv2:1.0.0 imageID: docker-pullable://istio/proxyv2#sha256:77915a0b8c88cce11f04caf88c9ee30300d5ba1fe13146ad5ece9abf8826204c lastState: terminated: containerID: docker://52299a80a0fa8949578397357861a9066ab0148ac8771058b83e4c59e422a029 exitCode: 255 finishedAt: 2018-08-15T11:44:27Z reason: Error startedAt: 2018-08-15T11:41:02Z name: istio-proxy ready: true restartCount: 1 state: running: startedAt: 2018-08-15T11:44:28Z hostIP: 10.32.96.27 initContainerStatuses: - containerID: docker://f267bb44b70d2d383ce3f9943ab4e917bb0a42ecfe17fe0ed294bde4d8284c58 image: istio/proxy_init:1.0.0 imageID: docker-pullable://istio/proxy_init#sha256:345c40053b53b7cc70d12fb94379e5aa0befd979a99db80833cde671bd1f9fad lastState: {} name: istio-init ready: true restartCount: 0 state: terminated: containerID: docker://f267bb44b70d2d383ce3f9943ab4e917bb0a42ecfe17fe0ed294bde4d8284c58 exitCode: 0 finishedAt: 2018-08-15T11:41:00Z reason: Completed startedAt: 2018-08-15T11:41:00Z phase: Running podIP: 10.32.19.61 qosClass: Burstable startTime: 2018-08-15T11:40:59Z Here is the example command when I got the error sleep.legacy -> httpbin.foo > kubectl exec $(kubectl get pod -l app=sleep -n legacy -o jsonpath={.items..metadata.name}) -c sleep -n legacy -- curl http://httpbin.foo:8000/ip -s -o /dev/null -w "%{http_code}\n" 000 command terminated with exit code 7 ** Here is the example command when I get success status: sleep.legacy -> httpbin.legacy ** > kubectl exec $(kubectl get pod -l app=sleep -n legacy -o jsonpath={.items..metadata.name}) -csleep -n legacy -- curl http://httpbin.legacy:8000/ip -s -o /dev/null -w "%{http_code}\n" 200 I have followed the instruction to ensure there is no mtls policy defined, etc. > kubectl get policies.authentication.istio.io --all-namespaces No resources found. > kubectl get meshpolicies.authentication.istio.io No resources found. > kubectl get destinationrules.networking.istio.io --all-namespaces -o yaml | grep "host:" host: istio-policy.istio-system.svc.cluster.local host: istio-telemetry.istio-system.svc.cluster.local
NVM, I think I found why. There is configuration being messed up in my part. If you take a look at the statsd address, it is defined with unrecognized hostname istio-statsd-prom-bridge.istio-system.istio-system:9125. I noticed that after looking at the proxy container being restarted/crashed multiple times.
Openshift/Kubernetes volumes_from
I'm trying to mimic volumes_from using Openshift/Kubernetes. I have container A packaging the app and container B server the packaged app. I cannot use init containers since I'm stuck on kubernetes 1.2. I've tried the postStart lifecycle hook, detailed here: How to mimic '--volumes-from' in Kubernetes But, Openshift/Kubernetes is always complaining that container A is contantly crashing because once it's done packaging, it exits. How do I get Openshift/Kubernetes to stop complaining about container A crashing and just accept that it finished it's job? Or is there another way of having one container build a package for another container to run? Thanks in advance for your time. Update 1: I don't have kubectl, but using oc describe pod myapp-2-prehook: me:~/Projects/myapp (master) $ oc describe pod myapp-2-prehook Name: myapp-2-prehook Namespace: myproject Node: my.host/my.ip Start Time: Tue, 01 Nov 2016 15:30:55 -1000 Labels: openshift.io/deployer-pod-for.name=myapp-2 Status: Failed IP: Controllers: <none> Containers: lifecycle: Container ID: docker://97a5272ebfa56f0c40fdc95094f13da06dba889049f2cc964fe3e89f61bd7792 Image: my.ip:5000/myproject/myapp#sha256:cde5739c5f2bdc8c25b1dd514f698c543cfb6c8b68c3f1afbc7760e11597fde9 Image ID: docker://3be476fec505e5b979bac69d327d4ffb53b3f568e85547c5b66c229948435f44 Port: Command: scripts/build.sh QoS Tier: cpu: BestEffort memory: BestEffort State: Terminated Reason: Error Exit Code: 1 Started: Tue, 01 Nov 2016 15:31:21 -1000 Finished: Tue, 01 Nov 2016 15:31:42 -1000 Ready: False Restart Count: 0 Environment Variables: CUSTOM_VAR1: custom_value1 OPENSHIFT_DEPLOYMENT_NAME: myapp OPENSHIFT_DEPLOYMENT_NAMESPACE: myproject Conditions: Type Status Ready False Volumes: default-token-goe98: Type: Secret (a volume populated by a Secret) SecretName: default-token-goe98 No events. Output of oc get pod assessor-2-prehook -o yaml: apiVersion: v1 kind: Pod metadata: annotations: openshift.io/deployment.name: myapp-2 openshift.io/scc: restricted creationTimestamp: 2016-11-02T01:30:55Z labels: openshift.io/deployer-pod-for.name: myapp-2 name: myapp-2-prehook namespace: myproject resourceVersion: "21512896" selfLink: /api/v1/namespaces/myproject/pods/myapp-2-prehook uid: ffcb7766-a09b-11e6-9053-005056a65cf8 spec: activeDeadlineSeconds: 21600 containers: - command: - scripts/build.sh env: - name: CUSTOM_VAR1 value: custom_value1 - name: OPENSHIFT_DEPLOYMENT_NAME value: myapp-2 - name: OPENSHIFT_DEPLOYMENT_NAMESPACE value: myproject image: my.ip:5000/myproject/myapp#sha256:cde5739c5f2bdc8c25b1dd514f698c543cfb6c8b68c3f1afbc7760e11597fde9 imagePullPolicy: IfNotPresent name: lifecycle resources: {} securityContext: privileged: false seLinuxOptions: level: s0:c21,c0 terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: default-token-goe98 readOnly: true dnsPolicy: ClusterFirst host: my.host imagePullSecrets: - name: default-dockercfg-srrog nodeName: my.host restartPolicy: Never securityContext: seLinuxOptions: level: s0:c21,c0 serviceAccount: default serviceAccountName: default terminationGracePeriodSeconds: 30 volumes: - name: default-token-goe98 secret: secretName: default-token-goe98 status: conditions: - lastProbeTime: null lastTransitionTime: 2016-11-02T01:31:49Z message: 'containers with unready status: [lifecycle]' reason: ContainersNotReady status: "False" type: Ready containerStatuses: - containerID: docker://97a5272ebfa56f0c40fdc95094f13da06dba889049f2cc964fe3e89f61bd7792 image: my.ip:5000/myproject/myapp#sha256:cde5739c5f2bdc8c25b1dd514f698c543cfb6c8b68c3f1afbc7760e11597fde9 imageID: docker://3be476fec505e5b979bac69d327d4ffb53b3f568e85547c5b66c229948435f44 lastState: {} name: lifecycle ready: false restartCount: 0 state: terminated: containerID: docker://97a5272ebfa56f0c40fdc95094f13da06dba889049f2cc964fe3e89f61bd7792 exitCode: 1 finishedAt: 2016-11-02T01:31:42Z reason: Error startedAt: 2016-11-02T01:31:21Z hostIP: 128.49.90.62 phase: Failed startTime: 2016-11-02T01:30:55Z