mongodb: restore replicaset after kubernetes scaling down - mongodb

I configured a replicatset correctly.
After having scaled down mongodb kubernetes pods, replicat set truned out to invalid status:
> rs.status();
{
"ok" : 0,
"errmsg" : "Our replica set config is invalid or we are not a member of it",
"code" : 93,
"codeName" : "InvalidReplicaSetConfig"
}
My configuration is:
> rs.config();
{
"_id" : "rs0",
"version" : 3,
"term" : 2,
"protocolVersion" : NumberLong(1),
"writeConcernMajorityJournalDefault" : true,
"members" : [
{
"_id" : 0,
"host" : "mongors-0.mongors-service.hes-all.svc:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 1,
"host" : "mongors-1.mongors-service.hes-all.svc:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"catchUpTimeoutMillis" : -1,
"catchUpTakeoverDelayMillis" : 30000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
},
"replicaSetId" : ObjectId("626fb63f211511c4dcf938ac")
}
}
configuration details seem right, but when I run rs.initiate, or rs.reconfig(cfg):
> rs.reconfig(config);
{
"topologyVersion" : {
"processId" : ObjectId("6347bdffe3c3303e6f325b9a"),
"counter" : NumberLong(1)
},
"ok" : 0,
"errmsg" : "New config is rejected :: caused by :: replSetReconfig should only be run on a writable PRIMARY. Current state REMOVED;",
"code" : 10107,
"codeName" : "NotWritablePrimary"
}
> rs.initiate();
{
"ok" : 0,
"errmsg" : "already initialized",
"code" : 23,
"codeName" : "AlreadyInitialized"
}
Any ideas?

Related

mongodb who is primary in replication

How can I display which one is the primary mongodb machine from a Secondary replication, without being forced to login to every machine and check?
Running ismaster command only provide that current machine is secondary
rs0:SECONDARY> db.runCommand("ismaster")
{
"hosts" : [
"dbRby1:27017",
"dbRby2:27017",
"dbKrstd1:27017"
],
"setName" : "rs0",
"setVersion" : 5,
"ismaster" : false,
"secondary" : true,
"me" : "dbRby1:27017",
"maxBsonObjectSize" : 16777216,
"maxMessageSizeBytes" : 48000000,
"maxWriteBatchSize" : 1000,
"localTime" : ISODate("2016-11-24T07:36:09.855Z"),
"maxWireVersion" : 4,
"minWireVersion" : 0,
"ok" : 1
}
or by using the rc.conf(), I can't see that either
rs0:SECONDARY> rs.conf()
{
"_id" : "rs0",
"version" : 5,
"protocolVersion" : NumberLong(1),
"members" : [
{
"_id" : 0,
"host" : "dbRby1:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 2,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 1,
"host" : "dbRby2:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 2,
"host" : "dbKrstd1:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : "majority",
"wtimeout" : 5000
},
"replicaSetId" : ObjectId("5811ec4c70c224f06fba884b")
}
}
rs.status() will give the wanted information as #Xenwar

mongo replica set - data not replicating

I setup a test replcia set using mongo v3.2 on a single node and the rs.status() command in the mongo shell seems to indicate that the replica set is established (output below).
However, when I insert data into the primary, I can see that it gets inserted into the primary but I cannot seem to run the find operation on any of the secondary processes (message given below)
message on secondary
Error: error: { "ok" : 0, "errmsg" : "not master and slaveOk=false", "code" : 13435 }
rs.status()
{
"set" : "set0",
"date" : ISODate("2016-07-21T19:53:41.882Z"),
"myState" : 1,
"term" : NumberLong(1),
"heartbeatIntervalMillis" : NumberLong(2000),
"members" : [
{
"_id" : 0,
"name" : "127.0.0.1:27049",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 2388,
"optime" : {
"ts" : Timestamp(1469130790, 2),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2016-07-21T19:53:10Z"),
"electionTime" : Timestamp(1469128700, 1),
"electionDate" : ISODate("2016-07-21T19:18:20Z"),
"configVersion" : 1,
"self" : true
},
{
"_id" : 1,
"name" : "127.0.0.1:27050",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 2132,
"optime" : {
"ts" : Timestamp(1469130790, 2),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2016-07-21T19:53:10Z"),
"lastHeartbeat" : ISODate("2016-07-21T19:53:40.964Z"),
"lastHeartbeatRecv" : ISODate("2016-07-21T19:53:40.823Z"),
"pingMs" : NumberLong(0),
"syncingTo" : "127.0.0.1:27049",
"configVersion" : 1
},
{
"_id" : 2,
"name" : "127.0.0.1:27051",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 2132,
"optime" : {
"ts" : Timestamp(1469130790, 2),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2016-07-21T19:53:10Z"),
"lastHeartbeat" : ISODate("2016-07-21T19:53:40.963Z"),
"lastHeartbeatRecv" : ISODate("2016-07-21T19:53:40.823Z"),
"pingMs" : NumberLong(0),
"syncingTo" : "127.0.0.1:27049",
"configVersion" : 1
}
],
"ok" : 1
}
output from rs.conf() on primary
{
"_id" : "set0",
"version" : 1,
"protocolVersion" : NumberLong(1),
"members" : [
{
"_id" : 0,
"host" : "127.0.0.1:27049",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 1,
"host" : "127.0.0.1:27050",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 2,
"host" : "127.0.0.1:27051",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
},
"replicaSetId" : ObjectId("57911ff0bd131f1aeb2ef878")
}
}
After I ran the following command on the secondary process, I was able to read the data from the secondary:
rs.slaveOk()
or
db.getMongo().setSlaveOk()
After running the following command from the Mongo shell, I was able to get the required documents from Mongo using the find command.

mongodb replica set is ok but does not replicate

I've created my mongodb replicaset and all is correct except is not replicate in remote host, but I've tried to access remote pc at the port 27017 and working properly.
I created the database on the remote PC to see if this solved but nothing, I have also inserted new records but nothing, Any ideas?
rs.status()
{
"set" : "meteor",
"date" : ISODate("2016-03-08T16:14:24.181Z"),
"myState" : 1,
"term" : NumberLong(3),
"heartbeatIntervalMillis" : NumberLong(2000),
"members" : [
{
"_id" : 0,
"name" : "172.27.10.13:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 6920,
"optime" : {
"ts" : Timestamp(1457453535, 2),
"t" : NumberLong(3)
},
"optimeDate" : ISODate("2016-03-08T16:12:15Z"),
"electionTime" : Timestamp(1457446744, 1),
"electionDate" : ISODate("2016-03-08T14:19:04Z"),
"configVersion" : 1,
"self" : true
}
],
"ok" : 1
}
rs.conf
rs.conf( rs.config(
meteor:PRIMARY> rs.config()
{
"_id" : "meteor",
"version" : 1,
"protocolVersion" : NumberLong(1),
"members" : [
{
"_id" : 0,
"host" : "172.27.10.13:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
}
}
}
According to your configuration settings you didn't add any other members (secondaries, arbiters etc.) to your replica-set configuration. Beacuse of this mongodb has no way of knowing where to replicat to.
Try adding your remote host to the replica-set configuration like this:
rs.add("your-remote-host:port")
See: https://docs.mongodb.org/manual/tutorial/expand-replica-set/

MongoDB 3.0.1 Replication Setup

I am using MongoDB 3.0.1 version. i am try to setup for mongodb replicaiton in
our machine. i have use three mongodb in septate machine for replication.
machine1 - master
machine2 - slave
machine3 - slave
I refer this url http://docs.mongodb.org/manual/tutorial/deploy-replica-set/
I have configure for every machine in mongodb.conf file
replSet = rs1
fork = true
and i added two members in master machine using this command
rs.add(192.168.1.2)
rs.add(192.168.1.3)
but i insert a document in master machine but not replicate other two slave machine. i did check slave machine throw the following error
> show dbs
2015-05-18T12:43:22.020+0530 E QUERY Error: listDatabases failed:{ "note" : "from execCommand", "ok" : 0, "errmsg" : "not master" }
at Error (<anonymous>)
at Mongo.getDBs (src/mongo/shell/mongo.js:47:15)
at shellHelper.show (src/mongo/shell/utils.js:630:33)
at shellHelper (src/mongo/shell/utils.js:524:36)
at (shellhelp2):1:1 at src/mongo/shell/mongo.js:47
>
> rs.conf()
2015-05-18T12:43:38.692+0530 E QUERY Error: Could not retrieve replica set config: {
"info" : "run rs.initiate(...) if not yet done for the set",
"ok" : 0,
"errmsg" : "no replset config has been received",
"code" : 94
}
at Function.rs.conf (src/mongo/shell/utils.js:1011:11)
at (shell):1:4 at src/mongo/shell/utils.js:1011
>
Please help me to solve the problem. thanks & Advance.
EDIT:
rs1:PRIMARY> rs.conf()
{
"_id" : "rs1",
"version" : 4,
"members" : [
{
"_id" : 0,
"host" : "analyzer-xubuntu:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : 0,
"votes" : 1
},
{
"_id" : 1,
"host" : "192.168.1.31:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 0.75,
"tags" : {
},
"slaveDelay" : 0,
"votes" : 1
},
{
"_id" : 2,
"host" : "192.168.1.33:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 0.5,
"tags" : {
},
"slaveDelay" : 0,
"votes" : 1
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatTimeoutSecs" : 10,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
}
}
}
rs1:PRIMARY> rs.status()
{
"set" : "rs1",
"date" : ISODate("2015-05-18T09:07:31.767Z"),
"myState" : 1,
"members" : [
{
"_id" : 0,
"name" : "analyzer-xubuntu:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 9236,
"optime" : Timestamp(1431939509, 2),
"optimeDate" : ISODate("2015-05-18T08:58:29Z"),
"electionTime" : Timestamp(1431931054, 2),
"electionDate" : ISODate("2015-05-18T06:37:34Z"),
"configVersion" : 4,
"self" : true
},
{
"_id" : 1,
"name" : "192.168.1.31:27017",
"health" : 1,
"state" : 0,
"stateStr" : "STARTUP",
"uptime" : 8953,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2015-05-18T09:07:29.831Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"pingMs" : 1,
"configVersion" : -2
},
{
"_id" : 2,
"name" : "192.168.1.33:27017",
"health" : 1,
"state" : 0,
"stateStr" : "STARTUP",
"uptime" : 8946,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2015-05-18T09:07:30.533Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"pingMs" : 1,
"configVersion" : -2
}
],
"ok" : 1
}
Let us go step by step.
Your MongoDB config is ok, the rest of work may be done in the shell. I will use machine names instead of IPs.
First connect to machine1, and run the following there:
> conf = {
_id: "rs1",
members:
[
{_id : 0, host : "machine1:27017"},
{_id : 1, host : "machine2:27017"},
{_id : 2, host : "machine3:27017"}
]
}
> rs.initiate(conf)
Then just run rs.slaveOk() on the secondaries. The secondaries will start replicating, and you will be able to query them and see your data, inserted from the primary.

Output of my high lockTime and high page_faults value also

we are using mongodb for our collections .
This is my stats of my development server .
My concerns are that there is high lockTime on my development server and also there are high page_faults value also .
Please tell me how can we resolve this ??
PRIMARY> db.serverStatus()
{
"host" : "ubsc-aio:27018",
"version" : "2.0.4",
"process" : "mongod",
"uptime" : 3278692,
"uptimeEstimate" : 3098346,
"localTime" : ISODate("2013-08-30T10:55:06.997Z"),
"globalLock" : {
"totalTime" : 3278692551767,
"lockTime" : 139516930214,
"ratio" : 0.0425526114483682,
"currentQueue" : {
"total" : 0,
"readers" : 0,
"writers" : 0
},
"activeClients" : {
"total" : 1,
"readers" : 1,
"writers" : 0
}
},
"mem" : {
"bits" : 64,
"resident" : 2009,
"virtual" : 23455,
"supported" : true,
"mapped" : 11420,
"mappedWithJournal" : 22840
},
"connections" : {
"current" : 162,
"available" : 7838
},
"extra_info" : {
"note" : "fields vary by platform",
"heap_usage_bytes" : 3645040,
"page_faults" : 4147570
},
"indexCounters" : {
"btree" : {
"accesses" : 902898,
"hits" : 901095,
"misses" : 1803,
"resets" : 0,
"missRatio" : 0.0019969033046922245
}
},
"backgroundFlushing" : {
"flushes" : 54639,
"total_ms" : 36709498,
"average_ms" : 671.8552316111203,
"last_ms" : 81,
"last_finished" : ISODate("2013-08-30T10:54:43.013Z")
},
"cursors" : {
"totalOpen" : 1,
"clientCursors_size" : 1,
"timedOut" : 7
},
"network" : {
"bytesIn" : 77779294441,
"bytesOut" : 314231714161,
"numRequests" : 189861092
},
"repl" : {
"setName" : "at",
"ismaster" : true,
"secondary" : false,
"hosts" : [
"localhost:27018",
"localhost:27017"
],
"arbiters" : [
"localhost:27019"
],
"primary" : "localhost:27018",
"me" : "localhost:27018"
},
"opcounters" : {
"insert" : 303294,
"query" : 133717078,
"update" : 59123588,
"delete" : 234256,
"getmore" : 48037783,
"command" : 125805489
},
"asserts" : {
"regular" : 0,
"warning" : 0,
"msg" : 0,
"user" : 16576,
"rollovers" : 0
},
"writeBacksQueued" : false,
"dur" : {
"commits" : 28,
"journaledMB" : 0.08192,
"writeToDataFilesMB" : 0.116123,
"compression" : 0.6743163821345669,
"commitsInWriteLock" : 0,
"earlyCommits" : 0,
"timeMs" : {
"dt" : 3000,
"prepLogBuffer" : 0,
"writeToJournal" : 25,
"writeToDataFiles" : 2,
"remapPrivateView" : 1
}
},
"ok" : 1
}