AWS ECS Fargate Logging: Facing issue with Firelens side-car container approach to push log files present inside application container to Cloudwatch - amazon-ecs

Question

I want to push log files which are present in a specific folder inside a container to Cloudwatch. For this, I tried Firelens logdriver but had no luck.
As mentioned here : https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/config-file-type-file
I created a custom docker image for fluentbit and deployed it as a side car container in task definition:
FROM public.ecr.aws/aws-observability/aws-for-fluent-bit:stable
COPY fluentbit.conf /extra.conf
fluentbit.conf (I tried exec for debugging purpose, aim is to use tail):
[INPUT]
Name exec
Tag exec_ls
Command ls /opt/apache-tomcat-8.5.72/my-app/logs
Interval_Sec 3
Interval_NSec 0
Buf_Size 8mb
[OUTPUT]
Name cloudwatch
Match *
region ap-south-1
log_group_name fluent-bit-adapter-logs
log_stream_prefix from-fluent-bit-
auto_create_group true
log_key log
This is my ECS Fargate task definition :
{
"ipcMode": null,
"executionRoleArn": "<role>",
"containerDefinitions": [
{
"dnsSearchDomains": null,
"environmentFiles": null,
"logConfiguration": {
"logDriver": "awsfirelens",
"secretOptions": null,
"options": {
"log_group_name": "/aws/ecs/containerinsights/$(ecs_cluster)/application",
"auto_create_group": "true",
"log_key": "log",
"log_stream_prefix": "log_stream_name",
"region": "ap-south-1",
"Name": "cloudwatch"
}
},
"entryPoint": null,
"portMappings": [
{
"hostPort": 8080,
"protocol": "tcp",
"containerPort": 8080
}
],
"command": null,
"linuxParameters": null,
"cpu": 0,
"environment": [],
"resourceRequirements": null,
"ulimits": null,
"dnsServers": null,
"mountPoints": [],
"workingDirectory": null,
"secrets": null,
"dockerSecurityOptions": null,
"memory": null,
"memoryReservation": null,
"volumesFrom": [],
"stopTimeout": null,
"image": "<image-url>",
"startTimeout": null,
"firelensConfiguration": null,
"dependsOn": null,
"disableNetworking": null,
"interactive": null,
"healthCheck": null,
"essential": true,
"links": null,
"hostname": null,
"extraHosts": null,
"pseudoTerminal": null,
"user": null,
"readonlyRootFilesystem": null,
"dockerLabels": null,
"systemControls": null,
"privileged": null,
"name": "my-app"
},
{
"dnsSearchDomains": null,
"environmentFiles": null,
"logConfiguration": {
"logDriver": "awslogs",
"secretOptions": null,
"options": {
"awslogs-group": "/ecs/log_router",
"awslogs-region": "ap-south-1",
"awslogs-create-group": "true",
"awslogs-stream-prefix": "firelens"
}
},
"entryPoint": null,
"portMappings": [],
"command": null,
"linuxParameters": null,
"cpu": 0,
"environment": [],
"resourceRequirements": null,
"ulimits": null,
"dnsServers": null,
"mountPoints": [],
"workingDirectory": null,
"secrets": null,
"dockerSecurityOptions": null,
"memory": null,
"memoryReservation": 50,
"volumesFrom": [],
"stopTimeout": null,
"image": "<image-url>",
"startTimeout": null,
"firelensConfiguration": {
"type": "fluentbit",
"options": {
"config-file-type": "file",
"config-file-value": "/extra.conf"
}
},
"dependsOn": null,
"disableNetworking": null,
"interactive": null,
"healthCheck": null,
"essential": true,
"links": null,
"hostname": null,
"extraHosts": null,
"pseudoTerminal": null,
"user": "0",
"readonlyRootFilesystem": null,
"dockerLabels": null,
"systemControls": null,
"privileged": null,
"name": "log_router"
}
],
"placementConstraints": [],
"memory": "8192",
"taskRoleArn": "<role>",
"compatibilities": [
"EC2",
"FARGATE"
],
"taskDefinitionArn": "<my-app arn>",
"family": "equbemi",
"requiresAttributes": [
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.ecr-auth"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.firelens.options.config.file"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.17"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.21"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.logging-driver.awsfirelens"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.task-iam-role"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-ecr-pull"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.task-eni"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.29"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.firelens.fluentbit"
}
],
"pidMode": null,
"requiresCompatibilities": [
"FARGATE"
],
"networkMode": "awsvpc",
"cpu": "4096",
"revision": 19,
"status": "ACTIVE",
"inferenceAccelerators": null,
"proxyConfiguration": null,
"volumes": []
}
For my application container, I've given logconfiguration as firelens and deployed a side-car container as mentioned in documentation. I tried the tail command in firelens config but didn't work. So just to troubleshoot, I tried exec and found out in firelens container logs that it is giving "File not found" exception. I assume it is trying to find the path in its own container(the side-car one) and not in the application container. I'm not sure how to make the firelens container access the application container. Am I missing anything here?
*

Answer 1

Here, the problem is that the fluent-bit container does not have access to the application container file system. You need to configure a volume so that the log path will be shared between both of these containers.
https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_data_volumes.html
Add a volume at the parent level and add this volume as a mount point in both of the container definitions:
{
...
"volumes": [
{
"fsxWindowsFileServerVolumeConfiguration": null,
"efsVolumeConfiguration": null,
"name": "log_path",
"host": {
"sourcePath": null
},
"dockerVolumeConfiguration": null
}
],
...
"containerDefinitions": [
{
...
"mountPoints": [
{
"readOnly": null,
"containerPath": "/var/log",
"sourceVolume": "log_path"
}
],
...
},
{
...
"mountPoints": [
{
"readOnly": null,
"containerPath": "/var/log",
"sourceVolume": "log_path"
}
],
...
}
]
...
}