How to consolidate Cloudwatch lambda log groups in a common log group - amazon-cloudwatchlogs

Question

My app has more than 100 lambdas on AWS as it works on a microservices pattern.
As configured by default, each lambda is writing on his own log group.
I'd like to get a view of the logs accross all my lambdas. I cannot use Logs Insights as I have more than 100 groups to search in.
Do you know how I can create a Log Group that would be an aggregation of all my lambda Log groups ?

Answer 1

Maybe there is a better way, but I ended up creating a Subscription Filter Lambda that copies all the logs to a single log group.
var aws = require("aws-sdk");
var zlib = require("zlib");
var { v4: uuid } = require("uuid");
const cloudwatch = new aws.CloudWatchLogs();
exports.handler = async function (input, context) {
const now = new Date();
const LOG_GROUP_NAME = "/aws/lambda/" + process.env.STAGE + "-main";
const LOG_STREAM_NAME =
now.getUTCFullYear() + "/" + now.getUTCMonth() + "/" + now.getUTCDay() + "/" + uuid();
try {
var zippedInput = new Buffer.from(input.awslogs.data, "base64");
const unzippedDataBuffer = await unzipLogs(zippedInput);
var awslogsData = JSON.parse(unzippedDataBuffer);
await cloudwatch
.createLogStream({
logGroupName: LOG_GROUP_NAME,
logStreamName: LOG_STREAM_NAME,
})
.promise();
await cloudwatch
.putLogEvents({
logGroupName: LOG_GROUP_NAME,
logStreamName: LOG_STREAM_NAME,
logEvents: awslogsData.logEvents.map((logEvent) => {
const { id, ...cleanEvent } = logEvent;
return cleanEvent;
}),
})
.promise();
} catch (err) {
console.error(error);
}
};
async function unzipLogs(zippedInput) {
return new Promise((resolve, reject) => {
zlib.gunzip(zippedInput, function (error, buffer) {
if (error) {
reject(error);
return;
}
resolve(buffer);
});
});
}