const client = new MongoClient(uri);
await client.connect();
await client
.db('mydb1')
.collection('foo');
const session = client.startSession();
const transactionOptions = {
readPreference: 'primary',
readConcern: { level: 'local' },
writeConcern: { w: 'majority' }
};
try {
await session.withTransaction(async () => {
const coll1 = client.db('mydb1').collection('foo');
await coll1.updateOne({user_id: 12344, paid: false }, { $set: { paid: true } } { session });
// long running computation after this line.
// what if another query deletes the document inserted above
// before this transaction completes.
await calls_third_party_payment_vendor_api_to_process_payment();
}, transactionOptions);
} finally {
await session.endSession();
await client.close();
}
What if the update document inside the transaction is simultaneously updated from an outside query before the transaction is committed?
What you have described is a transaction/operation conflict and
the operation blocks behind the transaction commit and infinitely retries with backoff logic until MaxTimeMS is reached
I wrote a report on MongoDB transactions, containing some examples in NodeJS too. if you are interested on the subject, I recommend you the paragraphs WiredTiger Cache and What happens when we create a multi-document transaction?
Related
What can potentially go wrong if you attempt to execute a query that is not part of the transaction inside of a transaction?
const session = client.startSession();
await session.withTransaction(async () => {
const coll1 = client.db('mydb1').collection('foo');
const coll2 = client.db('mydb2').collection('bar');
// Important:: You must pass the session to the operations
await coll1.insertOne({ abc: 1 } ); // does not have the session object
await coll2.insertOne({ xyz: 999 }, { session });
}, transactionOptions);
I am trying to find a similar solution to what node-rdkafka does for committing individual messages on success.
In node-rdkafka I was able to do call consumer.commit(message); after message processing succeeded. What is the equivalent in KafkaJS?
I have so far tried to call consumer.commitOffsets(...) inside eachMessage handler, but that didn't seem to commit.
I have code like this:
import { Kafka, logLevel } from 'kafkajs';
const kafka = new Kafka({
clientId: 'qa-topic',
brokers: [process.env.KAFKA_BOOTSTRAP_SERVER],
ssl: true,
logLevel: logLevel.INFO,
sasl: {
mechanism: 'plain',
username: process.env.KAFKA_CONSUMER_SASL_USERNAME,
password: process.env.KAFKA_CONSUMER_SASL_PASSWORD
}
});
const consumer = kafka.consumer({
groupId: process.env.KAFKA_CONSUMER_GROUP_ID
});
const run = async () => {
// Consuming
await consumer.connect()
await consumer.subscribe({ topic: 'my-topic', fromBeginning: true });
await consumer.run({
autoCommit: false,
eachMessage: async ({ topic, partition, message }) => {
try {
await processMyMessage(message);
// HOW DO I COMMIT THIS MESSAGE?
// The below doesn't seem to commit
// await consumer.commitOffsets([{ topic: 'my-topic', partition, offset:message.offset }]);
} catch (e) {
// log error, but do not commit message
}
},
})
}
I figured out how to do it. Can't use eachMessage handler, but instead use eachBatch which allows for more flexibility in control in how messages are committed
const run = async () => {
await consumer.connect();
await consumer.subscribe({ topic: 'my-topic', fromBeginning: true });
await consumer.run({
eachBatchAutoResolve: false,
eachBatch: async ({ batch, resolveOffset, isRunning, isStale }) => {
const promises = [];
logger.log(`Starting to process ${batch.messages?.length || 0} messages`);
for (const message of batch.messages) {
if (!isRunning() || isStale()) break;
promises.push(handleMessage(batch.topic, batch.partition, message, resolveOffset));
}
await Promise.all(promises);
},
});
};
Then inside handleMessage commit only those messages that succeeded
const handleMessage = async (topic, partition, message, resolveOffset) => {
try {
....
//Commit message if successful
resolveOffset(message.offset);
} catch(e) {
...
// Do not commit
}
As the documentation states: You can call consumer.commitOffsets only after consumer.run. You are trying to call it from within the run method, that is why it's not working for you.
Keep in mind that committing after each message increases the network traffic.
If that is a price you are willing to pay you can configure the auto-commit to take care of that for you by setting the autoCommitThreshold to 1.
I want to insert different collections inside a loop.
I already wrote this and it works once.
const client = new MongoClient(uri, {
useNewUrlParser: true,
useUnifiedTopology: true
});
let insertflowers = async (collectionname,query) => {
try {
await client.connect();
const database = client.db('flowers');
const collection = database.collection(collectionname);
return await collection.insertOne(query);
} finally {
await client.close();
}
}
insertflowers('alocasias',{name:'...'}).catch(console.dir);
What I want to do is put it inside a loop like this.
arrayofflowers.forEach( val => {
let flowerType = ...
insertflowers(flowerType,{name:'...'}).catch(console.dir);
});
But I get the following error
MongoError: Topology is closed, please connect
Thank you for reading
In short remove await client.close();
Check https://docs.mongodb.com/drivers/node/usage-examples/insertMany/ to insert bulk records at once.
You're in race condition so when insertflowers process is running in parallel connection is closed and opening.
So when you try to insert data connection is closed by another call to insertflowers.
const client = new MongoClient(uri, {
useNewUrlParser: true,
useUnifiedTopology: true
});
let connection;
const connect = async () => {
if (!connection) { // return connection if already connected
connection = await client.connect();
}
return connection;
});
let insertflowers = async (collectionname,query) => {
try {
const conn = await connect();
const database = conn.db('flowers');
const collection = database.collection(collectionname);
return await collection.insertOne(query);
} finally {
console.log('insertflowers completed');
// await client.close(); remove this
}
}
Another option - Not a good idea though
Make insertflowers is run the sync.
I am trying to use transactions to update multiple documents.
One being a loading sheet document [await sheet.save({ session });]
and other being an array of stock reservation records [await Stock.bulkWrite()].
const session = await mongoose.startSession();
session.startTransaction({
readPreference: 'primary',
readConcern: { level: 'local' },
writeConcern: { w: 'majority' },
});
let sheetAfterSave: any = null;
try {
sheetAfterSave = await sheet.save({ session });
records.forEach(async (el: any) => {
let updatedStockRecord = await Stock.bulkWrite(
[
{
updateOne: {
filter: {
index: el.index,
product: el.product,
batchNo: el.batchNo,
agency,
totalQuantity: { $gte: el.loadingTotal },
},
update: {
$push: {
reservations: {
loadingSheetId: sheetAfterSave._id,
reservedCaseQuantity: el.loadingCaseCount,
reservedUnitQuantity: el.loadingUnitCount,
reservedTotalQuantity: el.loadingTotal,
},
},
},
},
},
],
{
session: session,
}
);
});
await session.commitTransaction();
session.endSession();
} catch (error) {
console.log(error);
await session.abortTransaction();
session.endSession();
throw new Error(
`Error occured while trying to create a new loading sheet. ${error}`
);
}
In this operation loading sheet document gets saved in the database, but not the stock reservation records array.
It gives the error
[distribution] MongoError: Transaction 1 has been committed.
[distribution] at MessageStream.messageHandler (/app/node_modules/mongoose/node_modules/mongodb/lib/cmap/connection.js:263:20)
[distribution] at MessageStream.emit (node:events:376:20)
[distribution] at processIncomingData (/app/node_modules/mongoose/node_modules/mongodb/lib/cmap/message_stream.js:144:12)
[distribution] at MessageStream._write (/app/node_modules/mongoose/node_modules/mongodb/lib/cmap/message_stream.js:42:5)
[distribution] at writeOrBuffer (node:internal/streams/writable:388:12)
[distribution] at MessageStream.Writable.write (node:internal/streams/writable:333:10)
[distribution] at TLSSocket.ondata (node:internal/streams/readable:716:22)
[distribution] at TLSSocket.emit (node:events:376:20)
[distribution] at addChunk (node:internal/streams/readable:305:12)
[distribution] at readableAddChunk (node:internal/streams/readable:280:9)
[distribution] at TLSSocket.Readable.push (node:internal/streams/readable:219:10)
[distribution] at TLSWrap.onStreamRead (node:internal/stream_base_commons:192:23)
[distribution] [ERROR] 14:56:43 MongoError: Transaction 1 has been committed.
As I use the session in a failure like this the saved documents must be rolled back but its not happening here.
Am I missing something here?? Appreciate your help
Cheers
Do you see anything wrong with this code. I am trying to create a dynamic error inside the loop to make sure that all the the transactions are rolled back if any error occurs inside the loop?? Dynamic error outside the loop rolls back all the transactions perfectly but not the one inside the loop.
const session = await mongoose.startSession();
try {
await session.withTransaction(
async () => {
sheetAfterSave = await sheet.save({ session });
records.forEach(async (el: any) => {
let updatedStockRecord = await Stock.bulkWrite(
[
{
updateOne: {
filter: {
index: el.index,
product: el.product,
batchNo: el.batchNo,
agency,
totalQuantity: { $gte: el.loadingTotal },
},
update: {
$push: {
reservations: {
loadingSheetId: sheetAfterSave._id,
reservedCaseQuantity: el.loadingCaseCount,
reservedUnitQuantity: el.loadingUnitCount,
reservedTotalQuantity: el.loadingTotal,
},
},
},
},
},
],
{
session: session,
}
);
console.log('******************');
throw new Error('12/24/2020 ERROR INSIDE LOOP'); // MongoError: Transaction 1 has been committed.
});
throw new Error('12/24/2020 ERROR OUTSIDE LOOP'); // All transactions are rolled back perfectly
},
{
readPreference: 'primary',
readConcern: { level: 'local' },
writeConcern: { w: 'majority' },
}
);
} catch (error) {
console.log('ERROR BLOCK', error);
throw new Error(
`Error occured while trying to create a new loading sheet. ${error}`
);
} finally {
session.endSession();
await mongoose.connection.close();
console.log('************ FINALLY *****************');
}
I was able to solve the issue.
Problem was not with the below code
await session.commitTransaction(); (success)
session.endSession(); (failure)
} catch (error) { (entered)
await session.abortTransaction(); (invoked)
but it was with the records.forEach loop.
records.forEach(async (el: any) => {...});
inside the foreach when throwing an error it is not caught by the outermost try catch block since the content inside the loop is in a different functional context than the code outside of the loop.
Once I changed the loop from .forEach to
for (const el of records) {}
its working as expected.
Posting the answer in case if someone faces the same in the future. Thanks for the support guys :)
This mostly happens when one or more an update or db operation is not executed before the commit transaction is called. for example:
const session = await mongoose.startSession();
await session.withTransaction(async() => {
await User.findOneAndUpdate({_id: user._id}, {$set: {profile: profile._id}}, {session});
profile.set(body);
profile.user = user._id;
profile.save({session}); // THIS LINE WILL CAUSE SUCH ERROR BECAUSE IT RETURNS PROMISE
})
session.endSession();
return profile;
The right thing to do is to make sure operations are executed/await before the session transaction is committed as shown below.
const session = await mongoose.startSession();
await session.withTransaction(async() => {
await User.findOneAndUpdate({_id: user._id}, {$set: {profile: profile._id}}, {session});
profile.set(body);
profile.user = user._id;
await profile.save({session}); //AWAIT TO ENSURE COMPLETION OF EXECUTION
})
session.endSession();
return profile;
In the original question, ForEach() loop used is async operation which will not guarantee that all operations are completed before the transaction is committed.
Substituting it with a for..of loop which is somehow sync operation will guarantee a linear other of execution and error-out instantly if anything fails
The following sequence of calls/events will attempt to abort a committed transaction:
await session.commitTransaction(); (success)
session.endSession(); (failure)
} catch (error) { (entered)
await session.abortTransaction(); (invoked)
See https://docs.mongodb.com/manual/core/transactions/ for correct usage.
I'm trying to run this following transaction in a flutter application.
var db = Firestore.instance;
var chatReference = db
.collection('Messages')
.document(groupChatId)
.collection("Chats")
.document(docId);
var newChatReference = db
.collection('NewMessages')
.document(groupChatId);
Firestore.instance.runTransaction((transaction) async {
DocumentSnapshot snap;
snap = await transaction.get(newChatReference);
storedTS = snap.data['timestamp'];
//'timestamp' is stored as string in the db
//storedTS will be compared with a given string later on
//update 'content' field (string) with new value
await transaction.update(
chatReference, {
'timestamp': DateTime.now().millisecondsSinceEpoch.toString(),
'content': 'My updated content',
},
).then((val) {
print('updated');
}).catchError((e) {
print('updateErr: $e');
});
if (storedTS == myCondition) {
//if true then also update this other document. If not, just ignore it.
//During test if it IS true, both docs are updated as expected.
// If it is NOT true, then the first update completes, since the console
shows
//updated. But in the db the value remains the same, which means it was
//rolled back just after being updated (or so I guess).
await transaction.update(
newChatReference, {
'timestamp': DateTime.now().millisecondsSinceEpoch.toString(),
}
).then((val) {
print('newChat TS updated');
}).catchError((e) {
print('updateErr newChat TS: $e');
});
}
}).catchError((e) {
print("rtError: $e");
});
I get a print in the console saying 'updated' but nothing changes in the db in the collection from the first update operation. If I take out the if block in the second update, then it works as expected. Flabbergasted! Hope someone here can point out what I'm doing wrong.
Ok, after lots of digging around I learned that if you READ a doc in a tx, you MUST also do a WRITE on that doc. Beats me why it has to be like that, but for now we'll just have to live with it, I guess.