Error starting up a backup WiredTiger mongodb Database - mongodb

Question

I'm tasked to restore a WiredTiger mongodb backup on a new server, I've downloaded the folder to a path /data/db
But when
running sudo mongod --dbpath=/data/db on a new server, gave the following error.
{"t":{"$date":"2020-11-06T22:11:16.443+00:00"},"s":"W", "c":"ASIO", "id":22601, "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
{"t":{"$date":"2020-11-06T22:11:16.443+00:00"},"s":"I", "c":"NETWORK", "id":4648601, "ctx":"main","msg":"Implicit TCP FastOpen unavailable. If TCP FastOpen is required, set tcpFastOpenServer, tcpFastOpenClient, and tcpFastOpenQueueSize."}
{"t":{"$date":"2020-11-06T22:11:16.443+00:00"},"s":"I", "c":"STORAGE", "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":19032,"port":27017,"dbPath":"/data/db","architecture":"64-bit","host":"ip-172-31-4-37"}}
{"t":{"$date":"2020-11-06T22:11:16.443+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"4.4.1","gitVersion":"ad91a93a5a31e175f5cbf8c69561e788bbc55ce1","openSSLVersion":"OpenSSL 1.1.1f 31 Mar 2020","modules":[],"allocator":"tcmalloc","environment":{"distmod":"ubuntu2004","distarch":"x86_64","target_arch":"x86_64"}}}}
{"t":{"$date":"2020-11-06T22:11:16.443+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"Ubuntu","version":"20.04"}}}
{"t":{"$date":"2020-11-06T22:11:16.443+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"storage":{"dbPath":"/data/db"}}}}
{"t":{"$date":"2020-11-06T22:11:16.444+00:00"},"s":"I", "c":"STORAGE", "id":22270, "ctx":"initandlisten","msg":"Storage engine to use detected by data files","attr":{"dbpath":"/data/db","storageEngine":"wiredTiger"}}
{"t":{"$date":"2020-11-06T22:11:16.444+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=94061M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],"}}
{"t":{"$date":"2020-11-06T22:11:16.908+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1604700676:908863][19032:0x7fd255377c80], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 3 through 4"}}
{"t":{"$date":"2020-11-06T22:11:16.953+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1604700676:953627][19032:0x7fd255377c80], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 4 through 4"}}
{"t":{"$date":"2020-11-06T22:11:17.039+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1604700677:39694][19032:0x7fd255377c80], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 3/640 to 4/256"}}
{"t":{"$date":"2020-11-06T22:11:17.040+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1604700677:40016][19032:0x7fd255377c80], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 3 through 4"}}
{"t":{"$date":"2020-11-06T22:11:17.092+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1604700677:92204][19032:0x7fd255377c80], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 4 through 4"}}
{"t":{"$date":"2020-11-06T22:11:17.134+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1604700677:134313][19032:0x7fd255377c80], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (1604638812, 8737)"}}
{"t":{"$date":"2020-11-06T22:11:17.134+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1604700677:134351][19032:0x7fd255377c80], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (1604638812, 8737)"}}
{"t":{"$date":"2020-11-06T22:11:25.589+00:00"},"s":"I", "c":"STORAGE", "id":4795906, "ctx":"initandlisten","msg":"WiredTiger opened","attr":{"durationMillis":9145}}
{"t":{"$date":"2020-11-06T22:11:25.590+00:00"},"s":"I", "c":"RECOVERY", "id":23987, "ctx":"initandlisten","msg":"WiredTiger recoveryTimestamp","attr":{"recoveryTimestamp":{"$timestamp":{"t":1604638812,"i":8737}}}}
{"t":{"$date":"2020-11-06T22:11:25.591+00:00"},"s":"I", "c":"STORAGE", "id":22383, "ctx":"initandlisten","msg":"The size storer reports that the oplog contains","attr":{"numRecords":327031365,"dataSize":133132846247}}
{"t":{"$date":"2020-11-06T22:11:25.592+00:00"},"s":"I", "c":"STORAGE", "id":22386, "ctx":"initandlisten","msg":"Sampling the oplog to determine where to place markers for truncation"}
{"t":{"$date":"2020-11-06T22:11:25.592+00:00"},"s":"I", "c":"STORAGE", "id":22389, "ctx":"initandlisten","msg":"Sampling from the oplog to determine where to place markers for truncation","attr":{"from":{"$timestamp":{"t":1604570183,"i":3649}},"to":{"$timestamp":{"t":1604638812,"i":10667}}}}
{"t":{"$date":"2020-11-06T22:11:25.592+00:00"},"s":"I", "c":"STORAGE", "id":22390, "ctx":"initandlisten","msg":"Taking samples and assuming each oplog section contains","attr":{"numSamples":991,"containsNumRecords":3296963,"containsNumBytes":1342177280}}
{"t":{"$date":"2020-11-06T22:11:27.142+00:00"},"s":"I", "c":"STORAGE", "id":22393, "ctx":"initandlisten","msg":"Oplog sampling complete"}
{"t":{"$date":"2020-11-06T22:11:27.142+00:00"},"s":"I", "c":"STORAGE", "id":22382, "ctx":"initandlisten","msg":"WiredTiger record store oplog processing finished","attr":{"durationMillis":1550}}
{"t":{"$date":"2020-11-06T22:11:27.176+00:00"},"s":"I", "c":"STORAGE", "id":22262, "ctx":"initandlisten","msg":"Timestamp monitor starting"}
{"t":{"$date":"2020-11-06T22:11:27.177+00:00"},"s":"W", "c":"CONTROL", "id":22120, "ctx":"initandlisten","msg":"Access control is not enabled for the database. Read and write access to data and configuration is unrestricted","tags":["startupWarnings"]}
{"t":{"$date":"2020-11-06T22:11:27.177+00:00"},"s":"W", "c":"CONTROL", "id":22138, "ctx":"initandlisten","msg":"You are running this process as the root user, which is not recommended","tags":["startupWarnings"]}
{"t":{"$date":"2020-11-06T22:11:27.177+00:00"},"s":"W", "c":"CONTROL", "id":22140, "ctx":"initandlisten","msg":"This server is bound to localhost. Remote systems will be unable to connect to this server. Start the server with --bind_ip <address> to specify which IP addresses it should serve responses from, or with --bind_ip_all to bind to all interfaces. If this behavior is desired, start the server with --bind_ip 127.0.0.1 to disable this warning","tags":["startupWarnings"]}
{"t":{"$date":"2020-11-06T22:11:27.187+00:00"},"s":"W", "c":"CONTROL", "id":22167, "ctx":"initandlisten","msg":"You are running on a NUMA machine. We suggest launching mongod like this to avoid performance problems: numactl --interleave=all mongod [other options]","tags":["startupWarnings"]}
{"t":{"$date":"2020-11-06T22:11:27.187+00:00"},"s":"W", "c":"CONTROL", "id":22184, "ctx":"initandlisten","msg":"Soft rlimits too low","attr":{"currentValue":1024,"recommendedMinimum":64000},"tags":["startupWarnings"]}
{"t":{"$date":"2020-11-06T22:11:27.189+00:00"},"s":"I", "c":"STORAGE", "id":21005, "ctx":"initandlisten","msg":"Not restarting unfinished index builds because we are in standalone mode"}
{"t":{"$date":"2020-11-06T22:11:27.284+00:00"},"s":"F", "c":"CONTROL", "id":20573, "ctx":"initandlisten","msg":"Wrong mongod version","attr":{"error":"UPGRADE PROBLEM: Found an invalid featureCompatibilityVersion document (ERROR: BadValue: Invalid value for version, found 4.0, expected '4.4' or '4.2'. Contents of featureCompatibilityVersion document in admin.system.version: { _id: \"featureCompatibilityVersion\", version: \"4.0\" }. See https://docs.mongodb.com/master/release-notes/4.4-compatibility/#feature-compatibility.). If the current featureCompatibilityVersion is below 4.2, see the documentation on upgrading at https://docs.mongodb.com/master/release-notes/4.4/#upgrade-procedures."}}
{"t":{"$date":"2020-11-06T22:11:27.284+00:00"},"s":"I", "c":"REPL", "id":4784900, "ctx":"initandlisten","msg":"Stepping down the ReplicationCoordinator for shutdown","attr":{"waitTimeMillis":10000}}
{"t":{"$date":"2020-11-06T22:11:27.284+00:00"},"s":"I", "c":"COMMAND", "id":4784901, "ctx":"initandlisten","msg":"Shutting down the MirrorMaestro"}
{"t":{"$date":"2020-11-06T22:11:27.284+00:00"},"s":"I", "c":"SHARDING", "id":4784902, "ctx":"initandlisten","msg":"Shutting down the WaitForMajorityService"}
{"t":{"$date":"2020-11-06T22:11:27.284+00:00"},"s":"I", "c":"NETWORK", "id":20562, "ctx":"initandlisten","msg":"Shutdown: going to close listening sockets"}
{"t":{"$date":"2020-11-06T22:11:27.284+00:00"},"s":"I", "c":"NETWORK", "id":4784905, "ctx":"initandlisten","msg":"Shutting down the global connection pool"}
{"t":{"$date":"2020-11-06T22:11:27.284+00:00"},"s":"I", "c":"STORAGE", "id":4784906, "ctx":"initandlisten","msg":"Shutting down the FlowControlTicketholder"}
{"t":{"$date":"2020-11-06T22:11:27.284+00:00"},"s":"I", "c":"-", "id":20520, "ctx":"initandlisten","msg":"Stopping further Flow Control ticket acquisitions."}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":4784908, "ctx":"initandlisten","msg":"Shutting down the PeriodicThreadToAbortExpiredTransactions"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":4784934, "ctx":"initandlisten","msg":"Shutting down the PeriodicThreadToDecreaseSnapshotHistoryCachePressure"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"REPL", "id":4784909, "ctx":"initandlisten","msg":"Shutting down the ReplicationCoordinator"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"SHARDING", "id":4784910, "ctx":"initandlisten","msg":"Shutting down the ShardingInitializationMongoD"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"REPL", "id":4784911, "ctx":"initandlisten","msg":"Enqueuing the ReplicationStateTransitionLock for shutdown"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"-", "id":4784912, "ctx":"initandlisten","msg":"Killing all operations for shutdown"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"-", "id":4695300, "ctx":"initandlisten","msg":"Interrupted all currently running operations","attr":{"opsKilled":3}}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"COMMAND", "id":4784913, "ctx":"initandlisten","msg":"Shutting down all open transactions"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"REPL", "id":4784914, "ctx":"initandlisten","msg":"Acquiring the ReplicationStateTransitionLock for shutdown"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"INDEX", "id":4784915, "ctx":"initandlisten","msg":"Shutting down the IndexBuildsCoordinator"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"REPL", "id":4784916, "ctx":"initandlisten","msg":"Reacquiring the ReplicationStateTransitionLock for shutdown"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"REPL", "id":4784917, "ctx":"initandlisten","msg":"Attempting to mark clean shutdown"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"NETWORK", "id":4784918, "ctx":"initandlisten","msg":"Shutting down the ReplicaSetMonitor"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"SHARDING", "id":4784921, "ctx":"initandlisten","msg":"Shutting down the MigrationUtilExecutor"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"CONTROL", "id":4784925, "ctx":"initandlisten","msg":"Shutting down free monitoring"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"FTDC", "id":4784926, "ctx":"initandlisten","msg":"Shutting down full-time data capture"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":4784927, "ctx":"initandlisten","msg":"Shutting down the HealthLog"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":4784929, "ctx":"initandlisten","msg":"Acquiring the global lock for shutdown"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":4784930, "ctx":"initandlisten","msg":"Shutting down the storage engine"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":20282, "ctx":"initandlisten","msg":"Deregistering all the collections"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":22372, "ctx":"OplogVisibilityThread","msg":"Oplog visibility thread shutting down."}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":22261, "ctx":"initandlisten","msg":"Timestamp monitor shutting down"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":22317, "ctx":"initandlisten","msg":"WiredTigerKVEngine shutting down"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":22318, "ctx":"initandlisten","msg":"Shutting down session sweeper thread"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":22319, "ctx":"initandlisten","msg":"Finished shutting down session sweeper thread"}
{"t":{"$date":"2020-11-06T22:11:27.285+00:00"},"s":"I", "c":"STORAGE", "id":22320, "ctx":"initandlisten","msg":"Shutting down journal flusher thread"}
{"t":{"$date":"2020-11-06T22:11:27.286+00:00"},"s":"I", "c":"STORAGE", "id":22321, "ctx":"initandlisten","msg":"Finished shutting down journal flusher thread"}
{"t":{"$date":"2020-11-06T22:11:27.286+00:00"},"s":"I", "c":"STORAGE", "id":22322, "ctx":"initandlisten","msg":"Shutting down checkpoint thread"}
{"t":{"$date":"2020-11-06T22:11:27.286+00:00"},"s":"I", "c":"STORAGE", "id":22323, "ctx":"initandlisten","msg":"Finished shutting down checkpoint thread"}
{"t":{"$date":"2020-11-06T22:11:27.286+00:00"},"s":"I", "c":"STORAGE", "id":4795902, "ctx":"initandlisten","msg":"Closing WiredTiger","attr":{"closeConfig":"leak_memory=true,"}}
{"t":{"$date":"2020-11-06T22:11:28.544+00:00"},"s":"I", "c":"STORAGE", "id":4795901, "ctx":"initandlisten","msg":"WiredTiger closed","attr":{"durationMillis":1258}}
{"t":{"$date":"2020-11-06T22:11:28.544+00:00"},"s":"I", "c":"STORAGE", "id":22279, "ctx":"initandlisten","msg":"shutdown: removing fs lock..."}
{"t":{"$date":"2020-11-06T22:11:28.544+00:00"},"s":"I", "c":"-", "id":4784931, "ctx":"initandlisten","msg":"Dropping the scope cache for shutdown"}
{"t":{"$date":"2020-11-06T22:11:28.544+00:00"},"s":"I", "c":"CONTROL", "id":20565, "ctx":"initandlisten","msg":"Now exiting"}
{"t":{"$date":"2020-11-06T22:11:28.544+00:00"},"s":"I", "c":"CONTROL", "id":23138, "ctx":"initandlisten","msg":"Shutting down","attr":{"exitCode":62}}
running mongod also gives the error above.
how do I fix this?

Answer 1

You need to look at "s": "F" which is severity Fail.
After reading about indexes it seems the cause may have to do with index builds.
As it is advice in the log, upgrade your mongod from 4.0 to 4.4. See this document for how to do upgrade.