A better way to delete the data in a MongoDB collection but keeping the index - mongodb

Question

I was working on my unittests which for each cases the database should be reset.
Because that the indexes should be kept but the data should be cleared, what will be a faster way to do that reset?

Answer 1

I am using pymongo, not sure if the driver will heavily impacts the performance.
There are 2 ways that come up to my mind
Simply executes delete_many()
Drop the whole collection and recreate the indexes
I found that:
for collections containing < 25K entries, the first way is faster
for collections containing ~ 25K entries, two ways are similar
for collections containing < 25K entries, the second way is faster.
Below is my script to test and run.
import time
import pymongo
from bson import ObjectId
from extutils import exec_timing_result
client = pymongo.MongoClient("mongodb://192.168.50.33:27017/?readPreference=primary&ssl=false")
def test_drop_recreate(col):
col.drop()
col.create_index([("mike", pymongo.DESCENDING)])
def test_delete_many(col):
col.delete_many({})
def main():
db = client.get_database("_tests")
STEP = 1000
to_insert = []
for count in range(0, STEP * 101, STEP):
# Get the MongoDB collection
col = db.get_collection("a")
# Insert dummy data
to_insert.extend([{"_id": ObjectId(), "mike": "ABC"} for _ in range(STEP)])
if to_insert:
col.insert_many(to_insert)
# Record the execution time of dropping the databㄇse then recreate indexes of it
_start = time.time()
test_drop_recreate(col)
ms_drop = time.time() - _start
# Insert dummy data
if to_insert:
col.insert_many(to_insert)
# Record the execution time of simply executes `delete_many()`
_start = time.time()
test_delete_many(col)
ms_del = time.time() - _start
if ms_drop > ms_del:
print(f"{count},-{(ms_drop / ms_del) - 1:.2%}")
else:
print(f"{count},+{(ms_del / ms_drop) - 1:.2%}")
if __name__ == '__main__':
main()
After I ran this script a few times, I generated a graph to visualize the result using the output.
(Above 0) means deletion takes longer
(Below 0) means dropping and recreating takes longer
The value represents additional time consumed.
For example: +20 means deletion takes 20 times longer than drop & create.

Answer 2

I tried to "save" all the existing indexes and recreate them after the collection drop, but there are tons of edge cases and hidden parameters to the index object. Here is my shot:
def convert_key_to_list_index(son_index):
"""
Convert index SON object to a list that can be used to create the same index.
:param SON son_index: The output of "list_indexes()":
SON([
('v', 2),
('key', SON([('timestamp', 1.0)])),
('name', 'timestamp_1'),
('expireAfterSeconds', 3600.0)
])
:return list: List of tuples: (field, direction) to use for pymonog function create_index
"""
key_index_list = []
index_key_definitions = son_index["key"]
for field, direction in index_key_definitions.items():
item = (field, int(direction))
key_index_list.append(item)
return key_index_list
def drop_and_recreate_indexes(db, collection_name):
"""
Use list_indexes() and not index_information() to get the "key" as a SON object instead of regular dict,
Because the SON object is an ordered dict, which the order of the key of the index are important for the
recreation action.
"""
son_indexes = list(db[collection_name].list_indexes())
db[collection_name].drop()
# Re-create the collection indexes
for son_index in son_indexes:
# Skip the default index for the field "_id"
# Notice: For collection without any index except of the default "_id_" index,
# the collection will not be recreated (Recreate the index recreate the collection).
if son_index['name'] == '_id_':
continue
recreate_index(db, collection_name, son_index)
def recreate_index(db, collection_name, son_index):
"""
Get a SON object that represent an index from the db, and create the same index in the db.
:param pymongo.database.Database db:
:param str collection_name:
:param SON son_index: The object that returned from the call to pymongo.collection.Collection.list_indexes()
# Notice: Currently supported only these fields: ["name", "unique", "expireAfterSeconds"]
# For more fields/options see: https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html
# ?highlight=create_index#pymongo.collection.Collection.create_index
:return:
"""
list_index = convert_key_to_list_index(son_index)
# This dict is defined to send parameter if exists to the "create_index" function.
create_index_kwargs = {}
# "name" will always be part of the SON object that represent the index.
create_index_kwargs["name"] = son_index.get("name")
# "unique" may exist or not in the SON object
unique = son_index.get("unique")
if unique:
create_index_kwargs["unique"] = unique
# "expireAfterSeconds" may exist or not in the SON object
expire_after_seconds = son_index.get("expireAfterSeconds", None)
if expire_after_seconds:
create_index_kwargs["expireAfterSeconds"] = expire_after_seconds
db[collection_name].create_index(list_index, **create_index_kwargs)