Can't write to bind mount on ECS Fragate when using non-root user - amazon-ecs

I'm using ECS with Fargate and trying to create a bind mount on ephemeral storage but my user (id 1000) is unable to write to the volume.
According to the documentation, it should be possible.
However the documentation mentions:
By default, the volume permissions are set to 0755 and the owner as root. These permissions can be customized in the Dockerfile.
So in my Dockerfile I have
ARG PHP_VERSION=8.1.2-fpm-alpine3.15
FROM php:$PHP_VERSION as php_base
ENV APP_USER=app
ENV APP_USER_HOME=/home/app
ENV APP_USER_UID=1000
ENV APP_USER_GID=1000
ENV APP_HOME=/srv/app
# create the app user
RUN set -eux; \
addgroup -g $APP_USER_GID -S $APP_USER; \
adduser -S -D -h "$APP_USER_HOME" -u $APP_USER_UID -s /sbin/nologin -G $APP_USER -g $APP_USER $APP_USER
RUN set -eux; \
mkdir -p /var/run/php; \
chown -R ${APP_USER}:${APP_USER} /var/run/php; \
# TODO THIS IS A TEST
chmod 777 /var/run/php
# ...
FROM php_base as php_prod
# ...
VOLUME ["/var/run/php"]
USER $APP_USER
WORKDIR "${APP_HOME}"
ENTRYPOINT ["/usr/local/bin/docker-php-entrypoint"]
CMD ["php-fpm"]
And in my task definition I have:
{
"taskDefinitionArn": "arn:aws:ecs:us-east-1:999999999999:task-definition/app:2",
"containerDefinitions": [
{
"name": "app-php",
"image": "999999999999.dkr.ecr.us-east-1.amazonaws.com/php:latest",
"cpu": 0,
"portMappings": [],
"essential": true,
"environment": [
{
"name": "DATABASE_PORT",
"value": "3306"
},
{
"name": "DATABASE_USERNAME",
"value": "app"
},
{
"name": "DATABASE_NAME",
"value": "app"
},
{
"name": "DATABASE_HOST",
"value": "db.xxxxxxxxxxxx.us-east-1.rds.amazonaws.com"
}
],
"mountPoints": [
{
"sourceVolume": "php_socket",
"containerPath": "/var/run/php",
"readOnly": false
}
],
"volumesFrom": [],
"secrets": [
{
"name": "DATABASE_PASSWORD",
"valueFrom": "arn:aws:ssm:us-east-1:999999999999:parameter/db-password"
}
],
"readonlyRootFilesystem": false,
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "app",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "app"
}
},
"healthCheck": {
"command": [
"docker-healthcheck"
],
"interval": 10,
"timeout": 3,
"retries": 3,
"startPeriod": 15
}
},
{
"name": "app-proxy",
"image": "999999999999.dkr.ecr.us-east-1.amazonaws.com/proxy:latest",
"cpu": 0,
"portMappings": [
{
"containerPort": 80,
"hostPort": 80,
"protocol": "tcp"
}
],
"essential": true,
"environment": [],
"mountPoints": [
{
"sourceVolume": "php_socket",
"containerPath": "/var/run/php",
"readOnly": false
}
],
"volumesFrom": [],
"dependsOn": [
{
"containerName": "app-php",
"condition": "HEALTHY"
}
],
"readonlyRootFilesystem": false,
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "app",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "app"
}
},
"healthCheck": {
"command": [
"curl",
"-s",
"localhost/status-nginx"
],
"interval": 10,
"timeout": 3,
"retries": 3,
"startPeriod": 15
}
}
],
"family": "bnc-stage-remises-app",
"taskRoleArn": "arn:aws:iam::999999999999:role/app-task",
"executionRoleArn": "arn:aws:iam::999999999999:role/app-exec",
"networkMode": "awsvpc",
"revision": 2,
"volumes": [
{
"name": "php_socket",
"host": {}
}
],
"status": "ACTIVE",
"requiresAttributes": [
{
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"name": "ecs.capability.execution-role-awslogs"
},
{
"name": "com.amazonaws.ecs.capability.ecr-auth"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"name": "com.amazonaws.ecs.capability.task-iam-role"
},
{
"name": "ecs.capability.container-health-check"
},
{
"name": "ecs.capability.container-ordering"
},
{
"name": "ecs.capability.execution-role-ecr-pull"
},
{
"name": "ecs.capability.secrets.ssm.environment-variables"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
},
{
"name": "ecs.capability.task-eni"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.29"
}
],
"placementConstraints": [],
"compatibilities": [
"EC2",
"FARGATE"
],
"requiresCompatibilities": [
"FARGATE"
],
"cpu": "256",
"memory": "2048",
"registeredAt": "2022-02-15T15:54:47.452Z",
"registeredBy": "arn:aws:sts::999999999999:assumed-role/OrganizationAccountAccessRole/9999999999999999999",
"tags": [
{
"key": "Project",
"value": "project-name"
},
{
"key": "Environment",
"value": "stage"
},
{
"key": "ManagedBy",
"value": "Terraform"
},
{
"key": "Client",
"value": "ClientName"
},
{
"key": "Namespace",
"value": "client-name"
},
{
"key": "Name",
"value": "app"
}
]
}
However, in ECS I keep getting
2022-02-15T20:36:14.679Z [15-Feb-2022 20:36:14] ERROR: unable to bind listening socket for address '/var/run/php/php-fpm.sock': Permission denied (13) app-php
2022-02-15T20:36:14.679Z [15-Feb-2022 20:36:14] ERROR: unable to bind listening socket for address '/var/run/php/php-fpm.sock': Permission denied (13) app-php
2022-02-15T20:36:14.679Z [15-Feb-2022 20:36:14] ERROR: FPM initialization failed app-php
2022-02-15T20:36:14.679Z [15-Feb-2022 20:36:14] ERROR: FPM initialization failed app-php

Turns out /var/run is a symlink to /run in my container and ECS wasn't able to handle this. I changed my setup to use /run/php instead of /var/run/php and everything works perfectly.

Related

Database persistence in ECS Fargate

We are running a MongoDB container in ECS Fargate. The ECS services starts up fine and database is accessible.
But how do I make it persistent as we all know that container storage is ephemeral? I have tried to mount efs file system following AWS documentation. The service gets created and the task runs fine but I am unable to access the database anymore. As in, I am unable to even login.
As far as EFS goes, I have tried both versions - unencrypted as well as encrypted.
My Task definition for MongoDB is as follows: (have removed the part where username and password parameters are passed)
{
"taskDefinitionArn": "arn:aws:ecs:us-east-2:1234567890:task-definition/mongo_efs_test_1215:2",
"containerDefinitions": [
{
"name": "mongo_efs_container_1215",
"image": "public.ecr.aws/docker/library/mongo:latest",
"cpu": 0,
"portMappings": [
{
"containerPort": 27017,
"hostPort": 27017,
"protocol": "tcp"
}
],
"essential": true,
"entryPoint": [],
"command": [],
"environment": [],
"mountPoints": [
{
"sourceVolume": "efs-disk",
"containerPath": "/data/db"
}
],
"volumesFrom": [],
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "/ecs/mongo_efs_test_1215",
"awslogs-region": "us-east-2",
"awslogs-stream-prefix": "ecs"
}
}
}
],
"family": "mongo_efs_test_1215",
"taskRoleArn": "arn:aws:iam::1234567890:role/ecsTaskExecutionRole",
"executionRoleArn": "arn:aws:iam::1234567890:role/ecsTaskExecutionRole",
"networkMode": "awsvpc",
"revision": 2,
"volumes": [
{
"name": "efs-disk",
"efsVolumeConfiguration": {
"fileSystemId": "fs-dvveweo837af981fa",
"rootDirectory": "/",
"transitEncryption": "DISABLED",
"authorizationConfig": {
"iam": "DISABLED"
}
}
}
],
"status": "ACTIVE",
"requiresAttributes": [
{
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"name": "ecs.capability.execution-role-awslogs"
},
{
"name": "ecs.capability.efsAuth"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"name": "ecs.capability.efs"
},
{
"name": "com.amazonaws.ecs.capability.task-iam-role"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.25"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
},
{
"name": "ecs.capability.task-eni"
}
],
"placementConstraints": [],
"compatibilities": [
"EC2",
"FARGATE"
],
"requiresCompatibilities": [
"FARGATE"
],
"cpu": "256",
"memory": "512",
"runtimePlatform": {
"operatingSystemFamily": "LINUX"
},
"registeredAt": "2022-12-15T12:06:34.477Z",
"registeredBy": "arn:aws:iam::1234567890:root",
"tags": []
}
Can someone please guide me? I am struggling with this for a really long time. Any help is really appreciated.
Thank you.

AWS Cloudformation stuck in UPDATE_ROLLBACK_FAILED

I deploy my AWS Lambdas via AWS Serverless Application Model (SAM). One of my Lambdas uses Numpy which I reference via a 3rd party layer from Klayers by #keithRozario. I was using Klayers-python38-numpy:16 but discovered that it was deprecated after I deployed today which left my stack in an UPDATE_ROLLBACK_FAILED state.
One recommendation is to use Stack actions -> Continue update rollback from the AWS console; which I tried but it didn't work. The other solution is to delete the stack. However, this would be my first time deleting a stack and what I'd like to know is: if I delete my stack via the console, will my stack get recreated when I redeploy it? I've looked for answers to my question but I'm only finding references to deleting resources within the stack.
What I'd also like to know is, my stack is the first stack of many in an AWS CodePipeline, will my pipeline still work if I delete my stack? Further, will I experience anymore failed stacks as I proceed to subsequent stacks within my pipeline?
Lastly, the plan is to update to Klayers-python38-numpy:19 when I redeploy.
EDIT: as per #marcin
The problem is that the Klayers-python38-numpy:16, that is already deployed throughout my stack, is no longer available. I tried deploying a change to my code this morning, my pipeline failed during the CreateChangeSet step. The fact that this layer is no longer available is, I'm assuming, the reason my stack is unable to rollback.
My pipeline looks like this:
{
"pipeline": {
"name": "my-pipeline",
"roleArn": "arn:aws:iam::123456789:role/my-pipeline-CodePipelineExecutionRole-4O8PAUJGLXYZ",
"artifactStore": {
"type": "S3",
"location": "my-pipeline-buildartifactsbucket-62byf2xqaa8z"
},
"stages": [
{
"name": "Source",
"actions": [
{
"name": "SourceCodeRepo",
"actionTypeId": {
"category": "Source",
"owner": "ThirdParty",
"provider": "GitHub",
"version": "1"
},
"runOrder": 1,
"configuration": {
"Branch": "master",
"OAuthToken": "****",
"Owner": "hugo",
"Repo": "my-pipeline"
},
"outputArtifacts": [
{
"name": "SourceCodeAsZip"
}
],
"inputArtifacts": []
}
]
},
{
"name": "Build",
"actions": [
{
"name": "CodeBuild",
"actionTypeId": {
"category": "Build",
"owner": "AWS",
"provider": "CodeBuild",
"version": "1"
},
"runOrder": 1,
"configuration": {
"ProjectName": "my-pipeline"
},
"outputArtifacts": [
{
"name": "BuildArtifactAsZip"
}
],
"inputArtifacts": [
{
"name": "SourceCodeAsZip"
}
]
}
]
},
{
"name": "CI",
"actions": [
{
"name": "CreateChangeSet",
"actionTypeId": {
"category": "Deploy",
"owner": "AWS",
"provider": "CloudFormation",
"version": "1"
},
"runOrder": 1,
"configuration": {
"ActionMode": "CHANGE_SET_REPLACE",
"Capabilities": "CAPABILITY_IAM",
"ChangeSetName": "my-pipeline-ChangeSet-ci",
"ParameterOverrides": "{\n \"MyEnvironment\" : \"ci\"\n}\n",
"RoleArn": "arn:aws:iam::123456789:role/my-pipeline-CloudFormationExecutionRole-1O8GOB5C2VXYZ",
"StackName": "my-pipeline-ci",
"TemplatePath": "BuildArtifactAsZip::packaged.yaml"
},
"outputArtifacts": [],
"inputArtifacts": [
{
"name": "BuildArtifactAsZip"
}
]
},
{
"name": "ExecuteChangeSet",
"actionTypeId": {
"category": "Deploy",
"owner": "AWS",
"provider": "CloudFormation",
"version": "1"
},
"runOrder": 2,
"configuration": {
"ActionMode": "CHANGE_SET_EXECUTE",
"ChangeSetName": "my-pipeline-ChangeSet-ci",
"RoleArn": "arn:aws:iam::123456789:role/my-pipeline-CloudFormationExecutionRole-1O8GOB5C2VXYZ",
"StackName": "my-pipeline-ci"
},
"outputArtifacts": [
{
"name": "my-pipelineCIChangeSet"
}
],
"inputArtifacts": []
}
]
},
{
"name": "Staging",
"actions": [
{
"name": "CreateChangeSet",
"actionTypeId": {
"category": "Deploy",
"owner": "AWS",
"provider": "CloudFormation",
"version": "1"
},
"runOrder": 1,
"configuration": {
"ActionMode": "CHANGE_SET_REPLACE",
"Capabilities": "CAPABILITY_IAM",
"ChangeSetName": "my-pipeline-ChangeSet-staging",
"ParameterOverrides": "{\n \"MyEnvironment\" : \"staging\"\n}\n",
"RoleArn": "arn:aws:iam::123456789:role/my-pipeline-CloudFormationExecutionRole-1O8GOB5C2VXYZ",
"StackName": "my-pipeline-staging",
"TemplatePath": "BuildArtifactAsZip::packaged.yaml"
},
"outputArtifacts": [],
"inputArtifacts": [
{
"name": "BuildArtifactAsZip"
}
]
},
{
"name": "ExecuteChangeSet",
"actionTypeId": {
"category": "Deploy",
"owner": "AWS",
"provider": "CloudFormation",
"version": "1"
},
"runOrder": 2,
"configuration": {
"ActionMode": "CHANGE_SET_EXECUTE",
"ChangeSetName": "my-pipeline-ChangeSet-staging",
"RoleArn": "arn:aws:iam::123456789:role/my-pipeline-CloudFormationExecutionRole-1O8GOB5C2VXYZ",
"StackName": "my-pipeline-staging"
},
"outputArtifacts": [
{
"name": "my-pipelineStagingChangeSet"
}
],
"inputArtifacts": []
}
]
},
{
"name": "Prod",
"actions": [
{
"name": "DeploymentApproval",
"actionTypeId": {
"category": "Approval",
"owner": "AWS",
"provider": "Manual",
"version": "1"
},
"runOrder": 1,
"configuration": {},
"outputArtifacts": [],
"inputArtifacts": []
},
{
"name": "CreateChangeSet",
"actionTypeId": {
"category": "Deploy",
"owner": "AWS",
"provider": "CloudFormation",
"version": "1"
},
"runOrder": 2,
"configuration": {
"ActionMode": "CHANGE_SET_REPLACE",
"Capabilities": "CAPABILITY_IAM",
"ChangeSetName": "my-pipeline-ChangeSet-prod",
"ParameterOverrides": "{\n \"MyEnvironment\" : \"prod\"\n}\n",
"RoleArn": "arn:aws:iam::123456789:role/my-pipeline-CloudFormationExecutionRole-1O8GOB5C2VXYZ",
"StackName": "my-pipeline-prod",
"TemplatePath": "BuildArtifactAsZip::packaged.yaml"
},
"outputArtifacts": [],
"inputArtifacts": [
{
"name": "BuildArtifactAsZip"
}
]
},
{
"name": "ExecuteChangeSet",
"actionTypeId": {
"category": "Deploy",
"owner": "AWS",
"provider": "CloudFormation",
"version": "1"
},
"runOrder": 3,
"configuration": {
"ActionMode": "CHANGE_SET_EXECUTE",
"ChangeSetName": "my-pipeline-ChangeSet-prod",
"RoleArn": "arn:aws:iam::123456789:role/my-pipeline-CloudFormationExecutionRole-1O8GOB5C2VXYZ",
"StackName": "my-pipeline-prod"
},
"outputArtifacts": [
{
"name": "my-pipelineProdChangeSet"
}
],
"inputArtifacts": []
}
]
}
],
"version": 1
}
}
if I delete my stack via the console, will my stack get recreated when I redeploy it?
Yes. You can try to deploy same stack again, but probably you should investigate why it failed in the first place.
What I'd also like to know is, my stack is the first stack of many in an AWS CodePipeline, will my pipeline still work if I delete my stack?
Don't know, but probably not. Its use case specific and you haven't provide any info about the CP.
Further, will I experience anymore failed stacks as I proceed to subsequent stacks within my pipeline?
If one action fails, you can't proceed with further actions. Even if you could, other stacks can depend on the first one, and they will fail as well.

Error: Error response from daemon: invalid volume specification - Windows 8.1 Docker Toolbox

Error: Error response from daemon: invalid volume specification: 'C:/Users/Anthony/magento2-devbox:/C:/Users/Anthony/magento2-devbox'
I have had a google around on this but I cant see how this path has been assembled. Most paths exclude the : and i am also not sure why it has assembled this :/C:/ or whether this is just produced for the error message.
This is the YAML for the replica set;
{
"kind": "ReplicaSet",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "magento2-monolith-54cdd5b4b7",
"namespace": "default",
"selfLink": "/apis/extensions/v1beta1/namespaces/default/replicasets/magento2-monolith-54cdd5b4b7",
"uid": "e819bfbd-8820-11e9-a613-080027316036",
"resourceVersion": "22855",
"generation": 1,
"creationTimestamp": "2019-06-06T06:04:12Z",
"labels": {
"app.kubernetes.io/instance": "magento2",
"app.kubernetes.io/name": "monolith",
"pod-template-hash": "54cdd5b4b7"
},
"annotations": {
"deployment.kubernetes.io/desired-replicas": "1",
"deployment.kubernetes.io/max-replicas": "1",
"deployment.kubernetes.io/revision": "1"
},
"ownerReferences": [
{
"apiVersion": "apps/v1",
"kind": "Deployment",
"name": "magento2-monolith",
"uid": "9ec9d23e-8691-11e9-a3dd-080027316036",
"controller": true,
"blockOwnerDeletion": true
}
]
},
"spec": {
"replicas": 1,
"selector": {
"matchLabels": {
"app.kubernetes.io/instance": "magento2",
"app.kubernetes.io/name": "monolith",
"pod-template-hash": "54cdd5b4b7"
}
},
"template": {
"metadata": {
"creationTimestamp": null,
"labels": {
"app.kubernetes.io/instance": "magento2",
"app.kubernetes.io/name": "monolith",
"pod-template-hash": "54cdd5b4b7"
}
},
"spec": {
"volumes": [
{
"name": "nginx-config-volume",
"configMap": {
"name": "magento2-monolith-nginx-config",
"defaultMode": 420
}
},
{
"name": "varnish-config-volume",
"configMap": {
"name": "magento2-monolith-varnish-config",
"defaultMode": 420
}
},
{
"name": "code",
"hostPath": {
"path": "C:/Users/Anthony/magento2-devbox",
"type": ""
}
}
],
"containers": [
{
"name": "monolith",
"image": "magento2-monolith:dev",
"ports": [
{
"containerPort": 8050,
"protocol": "TCP"
}
],
"env": [
{
"name": "DEVBOX_ROOT",
"value": "C:/Users/Anthony/magento2-devbox"
},
{
"name": "COMPOSER_HOME",
"value": "C:/Users/Anthony/magento2-devbox/.composer"
},
{
"name": "MAGENTO_ROOT",
"value": "C:/Users/Anthony/magento2-devbox/magento"
},
{
"name": "MAGENTO_ROOT_HOST",
"value": "C:/Users/Anthony/magento2-devbox/magento"
},
{
"name": "DEVBOX_ROOT_HOST",
"value": "C:/Users/Anthony/magento2-devbox"
},
{
"name": "IS_WINDOWS_HOST",
"value": "0"
}
],
"resources": {},
"volumeMounts": [
{
"name": "code",
"mountPath": "C:/Users/Anthony/magento2-devbox"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "Never",
"securityContext": {
"privileged": true,
"procMount": "Default"
}
},
{
"name": "monolith-xdebug",
"image": "magento2-monolith:dev-xdebug",
"ports": [
{
"containerPort": 8002,
"protocol": "TCP"
}
],
"env": [
{
"name": "DEVBOX_ROOT",
"value": "C:/Users/Anthony/magento2-devbox"
},
{
"name": "COMPOSER_HOME",
"value": "C:/Users/Anthony/magento2-devbox/.composer"
},
{
"name": "MAGENTO_ROOT",
"value": "C:/Users/Anthony/magento2-devbox/magento"
},
{
"name": "MAGENTO_ROOT_HOST",
"value": "C:/Users/Anthony/magento2-devbox/magento"
},
{
"name": "DEVBOX_ROOT_HOST",
"value": "C:/Users/Anthony/magento2-devbox"
},
{
"name": "IS_WINDOWS_HOST",
"value": "0"
}
],
"resources": {},
"volumeMounts": [
{
"name": "code",
"mountPath": "C:/Users/Anthony/magento2-devbox"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "Never",
"securityContext": {
"privileged": true,
"procMount": "Default"
}
},
{
"name": "nginx",
"image": "nginx:1.9",
"resources": {},
"volumeMounts": [
{
"name": "code",
"mountPath": "C:/Users/Anthony/magento2-devbox"
},
{
"name": "nginx-config-volume",
"mountPath": "/etc/nginx/nginx.conf",
"subPath": "nginx.conf"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent",
"securityContext": {
"privileged": true,
"procMount": "Default"
}
},
{
"name": "varnish",
"image": "million12/varnish",
"env": [
{
"name": "VCL_CONFIG",
"value": "/etc/varnish/magento.vcl"
},
{
"name": "VARNISHD_PARAMS",
"value": "-a 0.0.0.0:6081"
}
],
"resources": {},
"volumeMounts": [
{
"name": "varnish-config-volume",
"mountPath": "/etc/varnish/magento.vcl",
"subPath": "varnish.vcl"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "Always"
}
],
"restartPolicy": "Always",
"terminationGracePeriodSeconds": 30,
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"schedulerName": "default-scheduler"
}
}
},
"status": {
"replicas": 1,
"fullyLabeledReplicas": 1,
"observedGeneration": 1
}
}
I am new to Docker/Kubernetes after coming over from Vagrant so i do not know where to start. The information i have is from the web browser dashboard
Path is probably not converted to unix style. Since docker 1.9.0, Windows paths are not automatically converted (eg. C:\Users to /c/Users).
So your path should be like :
{
"name": "DEVBOX_ROOT",
"value": "/c/Users/Anthony/magento2-devbox"
}

Running two containers in a single pod via kubectl run does not terminate the pod with two containers

Trying to run a two docker containers in one kubernetes pod. The problem we face is that we have 2 images/containers in a Kubernetes pod whereas we used to have 1 image for this pod previously where it was working perfectly and shutting down the pod gracefully. Now the problem is that with 2 images or containers within the same pod both images are not shutting down at the same time gracefully.
We are doing integration testing of our container against the database which is MongoDB, our container terminates but MongoDB fails to terminate here and continues running
while getopts n:c:l:i:d:t: option
do
case "${option}"
in
n) Company_NAMESPACE=$OPTARG;;
c) Company_CONFIG=$OPTARG;;
l) Company_LICENSE=$OPTARG;;
i) Company_IMAGE=$OPTARG;;
d) Company_CONTAINER_NAME=$OPTARG;;
t) Company_TOKEN=$OPTARG;;
esac
done
kubectl run $Company_CONTAINER_NAME -n $Company_NAMESPACE --restart=Never --overrides='
{
"apiVersion": "v1",
"spec": {
"imagePullSecrets": [
{
"name": "Company-regsecret"
}
],
"initContainers": [
{
"name": "copy-configs",
"image": "busybox",
"command": ["sh", "-c", "cp /tmp/Company-config-volume/server/* /tmp/ng-rt/config/server/ 2>/dev/null || true; cp /tmp/Company-license-volume/licenses/* /tmp/ng-rt/config/licenses 2>/dev/null || true"],
"volumeMounts": [
{
"name": "Company-config-volume",
"mountPath": "mount_path"
},
{
"name": "'$Company_CONFIG'",
"mountPath": "mount_path"
},
{
"name": "Company-license-volume",
"mountPath": "mount_path"
},
{
"name": "'$Company_LICENSE'",
"mountPath": "mount_path"
}
]
}
],
"containers": [
{
"name": "mongodb-test",
"image": "mongo:3.6.8",
"command": [
"numactl",
"--interleave=all",
"mongod",
"--wiredTigerCacheSizeGB",
"0.1",
"--replSet",
"MainRepSet",
"--bind_ip_all"
],
"ports": [{
"containerPort": 27017
}],
"readinessProbe": {
"exec": {
"command": ["mongo", "--eval", "rs.initiate()"]
}
},
"terminationGracePeriodSeconds": 10
},
{
"env": [
{
"name": "AWS_ACCESS_KEY_ID",
"valueFrom": {
"secretKeyRef": {
"key": "AWS_ACCESS_KEY_ID",
"name": "aws-secrets"
}
}
},
{
"name": "AWS_SECRET_ACCESS_KEY",
"valueFrom": {
"secretKeyRef": {
"key": "AWS_SECRET_ACCESS_KEY",
"name": "aws-secrets"
}
}
},
{
"name": "AWS_REGION",
"valueFrom": {
"secretKeyRef": {
"key": "AWS_REGION",
"name": "aws-secrets"
}
}
},
{
"name": "BUILD_ID",
"valueFrom": {
"configMapKeyRef": {
"key": "BUILD_ID",
"name": "config"
}
}
}
],
"command": [
"sh",
"-c",
"mkdir -p mount_path 2\u003e/dev/null && npm test --skipConnectivityTestRethinkDB"
],
"name": "'$Company_CONTAINER_NAME'",
"image": "'$Company_IMAGE'",
"volumeMounts": [
{
"mountPath": "mount_path",
"name": "'$Company_CONFIG'"
},
{
"mountPath": "mount_path",
"name": "'$Company_LICENSE'"
}
]
}
],
"volumes": [
{
"name": "Company-config-volume",
"configMap": {
"name": "'$Company_CONFIG'"
}
},
{
"name": "'$Company_CONFIG'",
"emptyDir": {}
},
{
"name": "Company-license-volume",
"configMap": {
"name": "'$Company_LICENSE'"
}
},
{
"name": "'$Company_LICENSE'",
"emptyDir": {}
}
]
}
}
' --image=$Company_IMAGE -ti --rm --token=$Company_TOKEN
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "test-dev-58663",
"namespace": "d-int-company-dev-v2",
"labels": {
"run": "test-dev-58663"
},
"annotations": {
"kubernetes.io/psp": "nfs-provisioner"
}
},
"spec": {
"volumes": [
{
"name": "company-config-volume",
"configMap": {
"name": "test-core",
"defaultMode": 420
}
},
{
"name": "test-core",
"emptyDir": {}
},
{
"name": "company-license-volume",
"configMap": {
"name": "company-license",
"defaultMode": 420
}
},
{
"name": "company-license",
"emptyDir": {}
},
{
"name": "default-token-wqp5x",
"secret": {
"secretName": "default-token-wqp5x",
"defaultMode": 420
}
}
],
"initContainers": [
{
"name": "copy-configs",
"image": "busybox",
"command": [
"sh",
"-c",
"cp mount_path* mount_path 2>/dev/null || true; cp mount_path* mount_path 2>mount_path|| true"
],
"resources": {},
"volumeMounts": [
{
"name": "company-config-volume",
"mountPath": "mount_path"
},
{
"name": "test-core",
"mountPath": "mount_path"
},
{
"name": "company-license-volume",
"mountPath": "mount_path"
},
{
"name": "company-license",
"mountPath": "mount_path"
},
{
"name": "default-token-wqp5x",
"readOnly": true,
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "Always"
}
],
"containers": [
{
"name": "mongodb-test",
"image": "mongo:3.6.8",
"ports": [
{
"containerPort": 27017,
"protocol": "TCP"
}
],
"resources": {},
"volumeMounts": [
{
"name": "default-token-wqp5x",
"readOnly": true,
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
}
],
"readinessProbe": {
"exec": {
"command": [
"mongo",
"--eval",
"rs.initiate()"
]
},
"timeoutSeconds": 1,
"periodSeconds": 10,
"successThreshold": 1,
"failureThreshold": 3
},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent"
},
{
"name": "test-dev-58663",
"image": "image_path",
"command": [
"sh",
"-c",
"mkdir -p mount_path 2>/dev/null && npm test --skipConnectivityTestRethinkDB"
],
"env": [
{
"name": "AWS_ACCESS_KEY_ID",
"valueFrom": {
"secretKeyRef": {
"name": "aws-secrets",
"key": "AWS_ACCESS_KEY_ID"
}
}
},
{
"name": "AWS_SECRET_ACCESS_KEY",
"valueFrom": {
"secretKeyRef": {
"name": "aws-secrets",
"key": "AWS_SECRET_ACCESS_KEY"
}
}
},
{
"name": "AWS_REGION",
"valueFrom": {
"secretKeyRef": {
"name": "aws-secrets",
"key": "AWS_REGION"
}
}
},
{
"name": "BUILD_ID",
"valueFrom": {
"configMapKeyRef": {
"name": "tbsp-config",
"key": "BUILD_ID"
}
}
}
],
"resources": {},
"volumeMounts": [
{
"name": "test-core",
"mountPath": "mount_path"
},
{
"name": "company-license",
"mountPath": "mount_path"
},
{
"name": "default-token-wqp5x",
"readOnly": true,
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent"
}
],
"restartPolicy": "Never",
"terminationGracePeriodSeconds": 10,
"dnsPolicy": "ClusterFirst",
"serviceAccountName": "default",
"serviceAccount": "default",
"nodeName": "pc212",
"securityContext": {},
"imagePullSecrets": [
{
"name": "company-regsecret"
}
],
"schedulerName": "default-scheduler",
"tolerations": [
{
"key": "node.kubernetes.io/not-ready",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": 300
},
{
"key": "node.kubernetes.io/unreachable",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": 300
}
]
},
"status": {
"phase": "Pending",
"conditions": [
{
"type": "Initialized",
"status": "True",
"lastProbeTime": null,
"lastTransitionTime": "2019-05-10T12:50:39Z"
},
{
"type": "Ready",
"status": "False",
"lastProbeTime": null,
"lastTransitionTime": "2019-05-10T12:49:54Z",
"reason": "ContainersNotReady",
"message": "containers with unready status: [mongodb-test test-dev-58663]"
},
{
"type": "PodScheduled",
"status": "True",
"lastProbeTime": null,
"lastTransitionTime": "2019-05-10T12:49:54Z"
}
],
"hostIP": "10.10.2.12",
"podIP": "172.16.4.22",
"startTime": "2019-05-10T12:49:54Z",
"initContainerStatuses": [
{
"name": "copy-configs",
"state": {
"terminated": {
"exitCode": 0,
"reason": "Completed",
"startedAt": "2019-05-10T12:50:39Z",
"finishedAt": "2019-05-10T12:50:39Z",
"containerID": "docker://1bcd12f5848e32e82f7dfde8e245223345e87f70061b789cbbabc0f798436b59"
}
},
"lastState": {},
"ready": true,
"restartCount": 0,
"image": "busybox:latest",
"imageID": "docker-pullable://busybox#sha256:0b184b74edc63924be0d7f67d16f5afbcdbe61caa1aca9312ed3b5c57792f6c1",
"containerID": "docker://1bcd12f5848e32e82f7dfde8e245223345e87f70061b789cbbabc0f798436b59"
}
],
"containerStatuses": [
{
"name": "mongodb-test",
"state": {
"waiting": {
"reason": "PodInitializing"
}
},
"lastState": {},
"ready": false,
"restartCount": 0,
"image": "mongo:3.6.8",
"imageID": ""
},
{
"name": "test-dev-58663",
"state": {
"waiting": {
"reason": "PodInitializing"
}
},
"lastState": {},
"ready": false,
"restartCount": 0,
"image": "image_path",
"imageID": ""
}
],
"qosClass": "BestEffort"
}
}
Both the containers and the hosting pod should terminate gracefully.
You have pod consisting of two containers, one of them is supposed to run indefinitely, another one runs to completion. This is bad practice. You should split your pod into two separate things: Pod with Mongo and Job with your integration script. You have to write logic that watches Job to finish and then terminates both Pod and Job. You can do it like this:
kubectl apply -f integration-test.yaml
kubectl wait --for=condition=Complete --timeout=5m job/test
kubectl delete -f integration-test.yaml

KubeDNS x509: failed to load system roots and no roots provided but curl work

I encounter a trouble with the last version of kubernetes (1.5.1). I have a quiet non usual setup composed with 5 Redhat Enterprise server. 3 are nodes, 2 are masters. Both masters are on an etcd cluster, flannel had been also added in baremetal.
I have this looping log in the kube-DNS container :
Failed to list *api.Endpoints: Get https://*.*.*.33:443/api/v1/endpoints?resourceVersion=0: x509: failed to load system roots and no roots provided
I made a big number of tests concerning the certificate. Curl works with the same credentials perfectly. The generation has been made with the official recommandation of kubernetes.
This is my different files of configuration ( with just the censorship of the ip and hostname if needed).
kube-apiserver.yml
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "kube-apiserver",
"namespace": "kube-system",
"labels": {
"component": "kube-apiserver",
"tier": "control-plane"
}
},
"spec": {
"volumes": [
{
"name": "certs",
"hostPath": {
"path": "/etc/ssl/certs"
}
},
{
"name": "pki",
"hostPath": {
"path": "/etc/kubernetes"
}
}
],
"containers": [
{
"name": "kube-apiserver",
"image": "gcr.io/google_containers/kube-apiserver-amd64:v1.5.1",
"command": [
"/usr/local/bin/kube-apiserver",
"--v=0",
"--insecure-bind-address=127.0.0.1",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota",
"--service-cluster-ip-range=100.64.0.0/12",
"--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--client-ca-file=/etc/kubernetes/pki/ca.pem",
"--tls-cert-file=/etc/kubernetes/pki/apiserver.pem",
"--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--secure-port=5443",
"--allow-privileged",
"--advertise-address=X.X.X.33",
"--etcd-servers=http://X.X.X.33:2379,http://X.X.X.37:2379",
"--kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP"
],
"resources": {
"requests": {
"cpu": "250m"
}
},
"volumeMounts": [
{
"name": "certs",
"mountPath": "/etc/ssl/certs"
},
{
"name": "pki",
"readOnly": true,
"mountPath": "/etc/kubernetes/"
}
],
"livenessProbe": {
"httpGet": {
"path": "/healthz",
"port": 8080,
"host": "127.0.0.1"
},
"initialDelaySeconds": 15,
"timeoutSeconds": 15
}
}
],
"hostNetwork": true
}
}
kube-controlleur-manager.yml
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "kube-controller-manager",
"namespace": "kube-system",
"labels": {
"component": "kube-controller-manager",
"tier": "control-plane"
}
},
"spec": {
"volumes": [
{
"name": "pki",
"hostPath": {
"path": "/etc/kubernetes"
}
}
],
"containers": [
{
"name": "kube-controller-manager",
"image": "gcr.io/google_containers/kube-controller-manager-amd64:v1.5.1",
"command": [
"/usr/local/bin/kube-controller-manager",
"--v=0",
"--address=127.0.0.1",
"--leader-elect=true",
"--master=https://X.X.X.33",
"--cluster-name= kubernetes",
"--kubeconfig=/etc/kubernetes/kubeadminconfig",
"--root-ca-file=/etc/kubernetes/pki/ca.pem",
"--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem",
"--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem"
],
"resources": {
"requests": {
"cpu": "200m"
}
},
"volumeMounts": [
{
"name": "pki",
"readOnly": true,
"mountPath": "/etc/kubernetes/"
}
],
"livenessProbe": {
"httpGet": {
"path": "/healthz",
"port": 10252,
"host": "127.0.0.1"
},
"initialDelaySeconds": 15,
"timeoutSeconds": 15
}
}
],
"hostNetwork": true
}
}
kube-scheduler.yml
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "kube-scheduler",
"namespace": "kube-system",
"labels": {
"component": "kube-scheduler",
"tier": "control-plane"
}
},
"spec": {
"volumes": [
{
"name": "pki",
"hostPath": {
"path": "/etc/kubernetes"
}
}
],
"containers": [
{
"name": "kube-scheduler",
"image": "gcr.io/google_containers/kube-scheduler-amd64:v1.5.1",
"command": [
"/usr/local/bin/kube-scheduler",
"--v=0",
"--address=127.0.0.1",
"--leader-elect=true",
"--kubeconfig=/etc/kubernetes/kubeadminconfig",
"--master=https://X.X.X.33"
],
"resources": {
"requests": {
"cpu": "100m"
}
},
"volumeMounts": [
{
"name": "pki",
"readOnly": true,
"mountPath": "/etc/kubernetes/"
}
],
"livenessProbe": {
"httpGet": {
"path": "/healthz",
"port": 10251,
"host": "127.0.0.1"
},
"initialDelaySeconds": 15,
"timeoutSeconds": 15
}
}
],
"hostNetwork": true
}
}
haproxy.yml
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "haproxy",
"namespace": "kube-system",
"labels": {
"component": "kube-apiserver",
"tier": "control-plane"
}
},
"spec": {
"volumes": [
{
"name": "vol",
"hostPath": {
"path": "/etc/haproxy/haproxy.cfg"
}
}
],
"containers": [
{
"name": "haproxy",
"image": "docker.io/haproxy:1.7",
"resources": {
"requests": {
"cpu": "250m"
}
},
"volumeMounts": [
{
"name": "vol",
"readOnly": true,
"mountPath": "/usr/local/etc/haproxy/haproxy.cfg"
}
]
}
],
"hostNetwork": true
}
}
kubelet.service
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service
[Service]
WorkingDirectory=/var/lib/kubelet
EnvironmentFile=/etc/kubernetes/kubelet ExecStart=/usr/bin/kubelet \
$KUBELET_ADDRESS \
$KUBELET_POD_INFRA_CONTAINER \
$KUBELET_ARGS \
$KUBE_LOGTOSTDERR \
$KUBE_ALLOW_PRIV \
$KUBELET_NETWORK_ARGS \
$KUBELET_DNS_ARGS
Restart=on-failure
[Install]
WantedBy=multi-user.target
kubelet
KUBELET_ADDRESS="--address=0.0.0.0 --port=10250"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"
KUBELET_ARGS="--kubeconfig=/etc/kubernetes/kubeadminconfig --require-kubeconfig=true --pod-manifest-path=/etc/kubernetes/manifests"
KUBE_LOGTOSTDERR="--logtostderr=true --v=9"
KUBE_ALLOW_PRIV="--allow-privileged=true"
KUBELET_DNS_ARGS="--cluster-dns=100.64.0.10 --cluster-domain=cluster.local"
kubadminconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority: /etc/kubernetes/pki/ca.pem
server: https://X.X.X.33
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: admin
name: admin#kubernetes
- context:
cluster: kubernetes
user: kubelet
name: kubelet#kubernetes
current-context: admin#kubernetes
kind: Config
users:
- name: admin
user:
client-certificate: /etc/kubernetes/pki/admin.pem
client-key: /etc/kubernetes/pki/admin-key.pem
I already have seen most of the question relative from far to close to this question in the internet so i hope someone will have a hint to debug this.