MongoDB single node replica set faild on restart azure vm - mongodb

I'm running mongodb single node replica set on azure VM with Linux (ubuntu 20.04).
everything worked fine and i was able to start the replica set following the documentation from mongodb.
today i restarted the VM from the azure portal and the mongodb server seems to refuse getting connections.
I got this log from mongod.log :
"c":"CONTROL", "id":20698, "ctx":"main","msg":"***** SERVER RESTARTED *****"}
"c":"REPL", "id":21405, "ctx":"ReplCoord-0","msg":"Locally stored replica set configuration does not have a valid entry for the current node; waiting for reconfig or remote heartbeat","attr":{"error":{"code":74,"codeName":"NodeNotFound","errmsg":"No host described in new configuration with {version: 2, term: 5} for replica set rs0 maps to this node"},"localConfig":{"_id":"rs0","version":2,"term":5,"protocolVersion":1,"writeConcernMajorityJournalDefault":true,"members":[{"_id":0,"host":"cloud.visual-factories.com:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":1.0,"tags":{},"slaveDelay":0,"votes":1}],"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"catchUpTakeoverDelayMillis":30000,"getLastErrorModes":{},"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":{"$oid":"610b7cd7fb1f746f7ce41e65"}}}}}
"c":"REPL", "id":21392, "ctx":"ReplCoord-0","msg":"New replica set config in use","attr":{"config":{"_id":"rs0","version":2,"term":5,"protocolVersion":1,"writeConcernMajorityJournalDefault":true,"members":[{"_id":0,"host":"cloud.visual-factories.com:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":1.0,"tags":{},"slaveDelay":0,"votes":1}],"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"catchUpTakeoverDelayMillis":30000,"getLastErrorModes":{},"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":{"$oid":"610b7cd7fb1f746f7ce41e65"}}}}}
"c":"REPL", "id":21394, "ctx":"ReplCoord-0","msg":"This node is not a member of the config"}
"c":"REPL", "id":21358, "ctx":"ReplCoord-0","msg":"Replica set state transition","attr":{"newState":"REMOVED","oldState":"STARTUP"}}
"c":"-", "id":20883, "ctx":"conn1","msg":"Interrupted operation as its client disconnected","attr":{"opId":2573}}
"c":"NETWORK", "id":22989, "ctx":"conn1","msg":"Error sending response to client. Ending connection from remote","attr":{"error":{"code":9001,"codeName":"SocketException","errmsg":"Broken pipe"},"remote":"127.0.0.1:55402","connectionId":1}}
this is my replication config :
{
"_id" : "rs0",
"version" : 2,
"term" : 2,
"protocolVersion" : NumberLong(1),
"writeConcernMajorityJournalDefault" : true,
"members" : [
{
"_id" : 0,
"host" : "cloud.visual-factories.com:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"catchUpTimeoutMillis" : -1,
"catchUpTakeoverDelayMillis" : 30000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
},
"replicaSetId" : ObjectId("611419575ebb3f4f4ebe44ab")
}
}
and mongod.conf
# mongod.conf
# for documentation of all options, see:
# http://docs.mongodb.org/manual/reference/configuration-options/
# Where and how to store data.
storage:
dbPath: /var/lib/mongodb
journal:
enabled: true
# engine:
# mmapv1:
# wiredTiger:
# where to write logging data.
systemLog:
destination: file
logAppend: true
path: /var/log/mongodb/mongod.log
# network interfaces
net:
port: 27017
bindIp: 0.0.0.0
# how the process runs
processManagement:
timeZoneInfo: /usr/share/zoneinfo
security:
authorization: enabled
keyFile: /etc/keyfile.txt
#operationProfiling:
replication:
replSetName: "rs0"
#sharding:
## Enterprise-Only Options:
#auditLog:
#snmp:
any ideas why this is happen ?

Related

MongoDB - ReplicaSet - Failed to refresh key cache

I am deploying my first replica set on a Windows 10 machine with MongoDB 5.0.
For the purpose of the tutorial, I want all the servers running on my machine, on different ports.
A)
I created my first server with :
mongod --replSet rs0 --port 27018 --dbpath C:\data\R0S1
I opened another command line prompt and I connected to it with mongo --port 27018.
I iniated the set with the command rs.initiate().
B)
I created my second and third server with :
mongod --replSet rs0 --port 27019 --dbpath C:\data\R0S2
mongod --replSet rs0 --port 27020 --dbpath C:\data\R0S3
(I ran each of these commands in new command line prompts)
C)
I added the second and third server to the set by connecting to the primary server (on 27018
with mongo --port 27018) with :
rs.add("localhost:27019")
rs.add("localhost:27020")
At this stage, everything has worked as expected.
Running rs.conf() gives me :
{
"_id" : "rs0",
"version" : 5,
"term" : 1,
"members" : [
{
"_id" : 0,
"host" : "localhost:27018",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"secondaryDelaySecs" : NumberLong(0),
"votes" : 1
},
{
"_id" : 1,
"host" : "localhost:27019",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"secondaryDelaySecs" : NumberLong(0),
"votes" : 1
},
{
"_id" : 2,
"host" : "localhost:27020",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"secondaryDelaySecs" : NumberLong(0),
"votes" : 1
}
],
"protocolVersion" : NumberLong(1),
"writeConcernMajorityJournalDefault" : true,
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"catchUpTimeoutMillis" : -1,
"catchUpTakeoverDelayMillis" : 30000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
},
"replicaSetId" : ObjectId("61cc297329dca2f0673c2cff")
}
}
D) In a new command line prompt, I create my fourth server (with the idea of making it an arbiter) with :
mongod --replSet rs0 --port 30000 --dbpath C:\data\arb
Here I have a problem.
The log file for this new server contains the following error :
{"t":{"$date":"2021-12-29T10:50:51.767+01:00"},"s":"I", "c":"-",
"id":4939300, "ctx":"monitoring-keys-for-HMAC","msg":"Failed to
refresh key cache","attr":{"error":"NotYetInitialized: Cannot use
non-local read concern until replica set is finished
initializing.","nextWakeupMillis":19200}}
When connecting to the primary and trying to do "rs.addArb(localhost:30000)", the command blocks and does nothing, because of the above error on server on port 30000 I believe.
Do you have any ideas on what's going on and how I could solve my issue ?
---- edit 1 ----
below is my mongod.cfg file :
# mongod.conf
# for documentation of all options, see:
# http://docs.mongodb.org/manual/reference/configuration-options/
# Where and how to store data.
storage:
dbPath: C:\Program Files\MongoDB\Server\5.0\data
journal:
enabled: true
# engine:
# wiredTiger:
# where to write logging data.
systemLog:
destination: file
logAppend: true
path: C:\Program Files\MongoDB\Server\5.0\log\mongod.log
# network interfaces
net:
port: 27017
bindIp: 127.0.0.1
#processManagement:
#security:
#operationProfiling:
#replication:
#sharding:
## Enterprise-Only Options:
#auditLog:
#snmp:
it might be due to majority read concern introduced as part of mongodb 5.0 as default setting.
Check this if gives read concern as majority then change it to 2.
db.adminCommand({ "getDefaultRWConcern" : 1 })
This might be an issue with mongodb -https://www.mongodb.com/community/forums/t/single-node-replicaset-never-finishing-instanciating-error-cannot-use-non-local-read-concern-until-replica-set-is-finished-initializing/164815/3

mongodb replica set error "...replSetHeartbeat requires authentication..." after running rs.initiate()

I have 2 virtual machines with mongodb running on both. I have created a user and roles and added data for testing. Everything works until I try to set up a Replica set. When I run rs.initiate() I get the "...replSetHeartbeat requires authentication... " error seen below. I can connect to the "hulk" server from "hawkeye" manually by passing --host "nodeserver-hulk:27017. Any Ideas?
system
Ubuntu Server 18.04.4 LTS
Mongod v.4.2.3
rs.initiate command
rs.initiate(
{
_id: "r1",
version: 1,
members: [
{ _id: 0, host : "nodeserver-hulk:27017"},
{ _id: 1, host : "nodeserver-hawkeye:27017"}
]
}
)
rs.initiate error
{
"ok" : 0,
"errmsg" : "replSetInitiate quorum check failed because not all proposed set members responded affirmatively: nodeserver-hawkeye:27017 failed with command replSetHeartbeat requires authentication",
"code" : 74,
"codeName" : "NodeNotFound"
}
mongod.conf file(same for both servers)
# mongod.conf
# for documentation of all options, see:
# http://docs.mongodb.org/manual/reference/configuration-options/
# Where and how to store data.
storage:
dbPath: /mnt/mongo/data
journal:
enabled: true
# engine:
# mmapv1:
# wiredTiger:
# where to write logging data.
systemLog:
destination: file
logAppend: true
path: /var/log/mongodb/mongod.log
# network interfaces
net:
port: 27017
bindIp: 0.0.0.0
# how the process runs
processManagement:
timeZoneInfo: /usr/share/zoneinfo
#security:
security:
authorization: 'enabled'
#operationProfiling:
#replication:
replication:
replSetName: 'r1'
#sharding:
## Enterprise-Only Options:
#auditLog:
#snmp:
show users command
> show users
{
"_id" : "admin.james",
"userId" : UUID("3ed97f2e-de49-4b98-84c8-566b34805863"),
"user" : "james",
"db" : "admin",
"roles" : [
{
"role" : "userAdminAnyDatabase",
"db" : "admin"
},
{
"role" : "dbOwner",
"db" : "admin"
},
{
"role" : "clusterAdmin",
"db" : "admin"
},
{
"role" : "readWriteAnyDatabase",
"db" : "admin"
}
],
"mechanisms" : [
"SCRAM-SHA-1",
"SCRAM-SHA-256"
]
}
You need to add a keyFile in your security config so that each node can authenticate against the others.
https://docs.mongodb.com/manual/tutorial/enforce-keyfile-access-control-in-existing-replica-set/
security:
keyFile: <path-to-keyfile>

mongos returns empty collection when queried from different applications

We use sharded MongoDB configuration with 2 shards. There are several front applications, which retrieve data via mongos service (installed on each application server).
Problem is that at some point running the same query from Application server #1 will return empty collection, while running the same query from Application server #3 will return correct results.
The problem persists on Application server #1 until flush command is run. As documentation states:
"You should only need to run flushRouterConfig after movePrimary has
been run or after manually clearing the jumbo chunk flag."
But neither Primary was moved nor was flag cleared.
Any ideas why could this happen?
Config file
# mongos.conf
# for documentation of all options, see:
# http://docs.mongodb.org/manual/reference/configuration-options/
# where to write logging data.
systemLog:
destination: file
logAppend: false
logRotate: rename
path: /var/log/mongos.log
# how the process runs
processManagement:
fork: true # fork and run in background
pidFilePath: /var/run/mongos.pid # location of pidfile
# network interfaces
net:
port: 27017
bindIp: 127.0.0.1,94.130.167.133,192.168.0.1 # Listen to local interface only, comment to listen on all interfaces.
security:
keyFile: /var/lib/database/mongos/.keyfile
sharding:
configDB: myproject-config/192.168.0.15:27027,192.168.0.16:27017,192.168.0.17:27017
printShardingStatus()
MongoDB shell version: 3.2.8
connecting to: cron0.myproject.smapps.org:27017/test
config.locks collection empty or missing. be sure you are connected to a mongos
--- Sharding Status ---
sharding version: {
"_id" : 1,
"minCompatibleVersion" : 5,
"currentVersion" : 6,
"clusterId" : ObjectId("2c5bc3fd9b2385s9613b0la5")
}
shards:
{ "_id" : "myproject0", "host" : "myproject0/192.168.0.11:27017,192.168.0.12:27017", "state" : 1, "tags" : [ ] }
{ "_id" : "myproject1", "host" : "myproject1/192.168.0.13:27017,192.168.0.18:27017", "state" : 1, "tags" : [ "game" ] }
{ "_id" : "myproject2", "host" : "myproject2/192.168.0.26:27017,192.168.0.28:27017", "state" : 1, "tags" : [ "game" ] }
active mongoses:
"4.0.6" : 14
balancer:
Currently enabled: yes
Currently running: no
Failed balancer rounds in last 5 attempts: 0
Migration Results for the last 24 hours:
8 : Success
databases:
{ "_id" : "myproject", "primary" : "myproject0", "partitioned" : true, "version" : { "uuid" : BinData(4,"+YxjwrKzTuekDlKutwT1IA=="), "lastMod" : 1 } }
...

rs.initiate method not working on MongoDB

I'm running MongoDB 3.2.15 on Ubuntu 16.04 LTS
I can't execute rs.initiate();. It returns the following error
{
"ok" : 0,
"errmsg" : "assertion src/mongo/db/repl/replset_commands.cpp:275",
"code" : 8
}
opened mongo with the following arguments
mongod --config /etc/mongod.conf --replSet rs0
Configuration file :
storage:
dbPath: /var/lib/mongodb
journal:
enabled: true
systemLog:
destination: file
logAppend: true
path: /var/log/mongodb/mongod.log
net:
port: 27017
bindIp: 0.0.0.0
The log file :
2017-07-26T08:25:38.400+0000 I NETWORK [HostnameCanonicalizationWorker] Starting hostname canonicalization worker
2017-07-26T08:25:42.186+0000 I COMMAND [conn1] initiate : no configuration specified. Using a default configuration for the set
2017-07-26T08:25:42.186+0000 I - [conn1] Assertion failure h != "localhost" src/mongo/db/repl/replset_commands.cpp 275
Try this (success for me):
mongo
> config = {_id:"rs0", members:[ {_id:0, host:"127.0.0.1:27017"}]}
{
"_id" : "rs0",
"members" : [
{
"_id" : 0,
"host" : "127.0.0.1:27017"
}
]
}
> rs.initiate(config)
{ "ok" : 1 }
Reference: http://blog.csdn.net/Aegeaner/article/details/56277129

None of the hosts for the replica set could be contacted

I am having a problem with my sharded cluster.
I setup a new cluster, with 1 router, 2 replica set shards(2 nodes each), and a single 3 cluster config cluster.
I believe I setup everything correctly, created collections, added indexes, but when I go to insert or query data into the collections, I get the error:
Error: error: {
"ok" : 0,
"errmsg" : "None of the hosts for replica set configReplSet could be contacted.",
"code" : 71
}
configReplSet is my config replica set. It is accessible from the box, I was able to use a mongo shell to log into the primary of the RS.
Any help into what would cause this error would be greatly appreciated.
Here is my sh.status()
--- Sharding Status ---
sharding version: {
"_id" : 1,
"minCompatibleVersion" : 5,
"currentVersion" : 6,
"clusterId" : ObjectId("58a761728dfc0e1090b7c592")
}
shards:
{ "_id" : "rs0", "host" : "rs0/mdbshard-b1:27017,mdbshard-b1rep:27017" }
{ "_id" : "rs1", "host" : "rs1/mdbshard-b2rep:27018,mdbshard-b2:27017" }
active mongoses:
"3.2.12" : 1
balancer:
Currently enabled: yes
Currently running: no
Failed balancer rounds in last 5 attempts: 0
Migration Results for the last 24 hours:
No recent migrations
and my shard router config file
replication:
localPingThresholdMs: 15
sharding:
autoSplit: true
configDB: "configReplSet/mdbcfg-b1:27019,mdbcfg-b2:27019,mdbcfg-b3:27019"
chunkSize: 64
processManagement:
fork: true
systemLog:
destination: file
path: "/var/log/mongodb/mongodb.log"
logAppend: true
Please let me know if you need any other information, I would be happy to provide it.
Check if this helps Relevant Error