I have installed a on premise Replica set which has 6 data nodes and 1 arbiter. The nodes are spread across 3 data centers: 3 data nodes on DC1, 3 data nodes on DC2, arbiter on DC3. We chose this infrastructure so that we can achieve High Availability in case of a data center failure.
A few days ago the application could not connect to the Replica because it was in a weird state, two of the data nodes and the arbiter went down, but the other 4 data nodes were still running. I ran db.hello() on the primary node of the remaining nodes and the isWritablePrimary flag was false. It was strange because the remaining nodes could form a majority (4/7 nodes were running), the nodes elected a primary and every secondary was pointing to the same primary, but still, when i ran db.hello() on the primary it had isWritablePrimary: false, i couldn’t run any queries or insert any data on it.
When i tried to run a query on a collection it prompted the following error: “NotPrimaryNoSecondaryOk” which implies that the node i was trying to run the query was a seconday node, but guess what it wasn’t, every secondary node recognized the node on which i ran the queries as primary and so did the primary, it was recognizing itself as a primary node.
Regardless of the fact that the 3 nodes were down, the question is, why was the replica set not valid?
Show us the output of rs.status() from your “primary”?
I’m sorry, i don’t have the output anymore.
The problem happened again.
This is the output when i try to read from the primary node:
uncaught exception: Error: error: {
"topologyVersion" : {
"processId" : ObjectId("645cabac8243064457faa4a4"),
"counter" : NumberLong(48)
},
"operationTime" : Timestamp(1684863010, 1),
"ok" : 0,
"errmsg" : "not master and slaveOk=false",
"code" : 13435,
"codeName" : "NotPrimaryNoSecondaryOk",
"$clusterTime" : {
"clusterTime" : Timestamp(1684863052, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
}
} :
_getErrorWithCode@src/mongo/shell/utils.js:25:13
DBCommandCursor@src/mongo/shell/query.js:703:15
DBQuery.prototype._exec@src/mongo/shell/query.js:112:28
DBQuery.prototype.hasNext@src/mongo/shell/query.js:287:5
DBCollection.prototype.findOne@src/mongo/shell/collection.js:260:10
@(shell):1:1
This is the output when I run rs.hello():
replica-set-0:PRIMARY> rs.hello()
{
"topologyVersion" : {
"processId" : ObjectId("645cabac8243064457faa4a4"),
"counter" : NumberLong(48)
},
"hosts" : [
"c1-mongoshdb01:27017",
"c2-mongoshdb02:27017",
"c2-mongoshdb03:27017",
"c1-mongoshrd01:27017",
"c1-mongoshrd02:27017",
"c2-mongoshrd03:27017"
],
"arbiters" : [
"nxt-db-arb:27017"
],
"setName" : "replica-set-0",
"setVersion" : 6,
"isWritablePrimary" : false,
"secondary" : true,
"primary" : "c2-mongoshdb02:27017",
"me" : "c2-mongoshdb02:27017",
"electionId" : ObjectId("7fffffff0000000000007b2b"),
"lastWrite" : {
"opTime" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"lastWriteDate" : ISODate("2023-05-23T17:30:10Z"),
"majorityOpTime" : {
"ts" : Timestamp(1684862963, 1),
"t" : NumberLong(31529)
},
"majorityWriteDate" : ISODate("2023-05-23T17:29:23Z")
},
"maxBsonObjectSize" : 16777216,
"maxMessageSizeBytes" : 48000000,
"maxWriteBatchSize" : 100000,
"localTime" : ISODate("2023-05-24T13:42:35.787Z"),
"logicalSessionTimeoutMinutes" : 30,
"connectionId" : 2861657,
"minWireVersion" : 0,
"maxWireVersion" : 9,
"readOnly" : false,
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1684863052, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
},
"operationTime" : Timestamp(1684863010, 1)
}
And this is the rs.status() output:
{
"set" : "replica-set-0",
"date" : ISODate("2023-05-24T13:44:09.778Z"),
"myState" : 1,
"term" : NumberLong(31531),
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"majorityVoteCount" : 4,
"writeMajorityCount" : 4,
"votingMembersCount" : 7,
"writableVotingMembersCount" : 6,
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1684862963, 1),
"t" : NumberLong(31529)
},
"lastCommittedWallTime" : ISODate("2023-05-23T17:29:23.122Z"),
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1684862963, 1),
"t" : NumberLong(31529)
},
"readConcernMajorityWallTime" : ISODate("2023-05-23T17:29:23.122Z"),
"appliedOpTime" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"durableOpTime" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"lastAppliedWallTime" : ISODate("2023-05-23T17:30:10.839Z"),
"lastDurableWallTime" : ISODate("2023-05-23T17:30:10.839Z")
},
"lastStableRecoveryTimestamp" : Timestamp(1684862963, 1),
"electionCandidateMetrics" : {
"lastElectionReason" : "electionTimeout",
"lastElectionDate" : ISODate("2023-05-23T17:30:52.026Z"),
"electionTerm" : NumberLong(31531),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(1684862963, 1),
"t" : NumberLong(31529)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1684862981, 1),
"t" : NumberLong(31530)
},
"numVotesNeeded" : 4,
"priorityAtElection" : 1,
"electionTimeoutMillis" : NumberLong(10000),
"priorPrimaryMemberId" : 2,
"targetCatchupOpTime" : {
"ts" : Timestamp(1684862981, 1),
"t" : NumberLong(31530)
}
},
"electionParticipantMetrics" : {
"votedForCandidate" : true,
"electionTerm" : NumberLong(31528),
"lastVoteDate" : ISODate("2023-05-22T17:36:31.798Z"),
"electionCandidateMemberId" : 0,
"voteReason" : "",
"lastAppliedOpTimeAtElection" : {
"ts" : Timestamp(1684776975, 1),
"t" : NumberLong(31527)
},
"maxAppliedOpTimeInSet" : {
"ts" : Timestamp(1684776975, 1),
"t" : NumberLong(31527)
},
"priorityAtElection" : 1
},
"members" : [
{
"_id" : 0,
"name" : "c1-mongoshdb01:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 72838,
"optime" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"optimeDurable" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"optimeDate" : ISODate("2023-05-23T17:30:10Z"),
"optimeDurableDate" : ISODate("2023-05-23T17:30:10Z"),
"lastAppliedWallTime" : ISODate("2023-05-23T17:30:10.839Z"),
"lastDurableWallTime" : ISODate("2023-05-23T17:30:10.839Z"),
"lastHeartbeat" : ISODate("2023-05-24T13:44:08.074Z"),
"lastHeartbeatRecv" : ISODate("2023-05-24T13:44:08.173Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "c2-mongoshdb02:27017",
"syncSourceId" : 1,
"infoMessage" : "",
"configVersion" : 6,
"configTerm" : 31530
},
{
"_id" : 1,
"name" : "c2-mongoshdb02:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 1140989,
"optime" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"optimeDate" : ISODate("2023-05-23T17:30:10Z"),
"lastAppliedWallTime" : ISODate("2023-05-23T17:30:10.839Z"),
"lastDurableWallTime" : ISODate("2023-05-23T17:30:10.839Z"),
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "syncing from: c2-mongoshdb03:27017",
"electionTime" : Timestamp(1684863052, 1),
"electionDate" : ISODate("2023-05-23T17:30:52Z"),
"configVersion" : 6,
"configTerm" : 31530,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 2,
"name" : "c2-mongoshdb03:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 72838,
"optime" : {
"ts" : Timestamp(1684862981, 1),
"t" : NumberLong(31530)
},
"optimeDurable" : {
"ts" : Timestamp(1684862963, 1),
"t" : NumberLong(31529)
},
"optimeDate" : ISODate("2023-05-23T17:29:41Z"),
"optimeDurableDate" : ISODate("2023-05-23T17:29:23Z"),
"lastAppliedWallTime" : ISODate("2023-05-23T17:29:41.665Z"),
"lastDurableWallTime" : ISODate("2023-05-23T17:29:23.122Z"),
"lastHeartbeat" : ISODate("2023-05-24T13:44:08.237Z"),
"lastHeartbeatRecv" : ISODate("2023-05-24T13:44:08.237Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1684862980, 1),
"electionDate" : ISODate("2023-05-23T17:29:40Z"),
"configVersion" : 6,
"configTerm" : 31530
},
{
"_id" : 3,
"name" : "c1-mongoshrd01:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 72838,
"optime" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"optimeDurable" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"optimeDate" : ISODate("2023-05-23T17:30:10Z"),
"optimeDurableDate" : ISODate("2023-05-23T17:30:10Z"),
"lastAppliedWallTime" : ISODate("2023-05-23T17:30:10.839Z"),
"lastDurableWallTime" : ISODate("2023-05-23T17:30:10.839Z"),
"lastHeartbeat" : ISODate("2023-05-24T13:44:08.074Z"),
"lastHeartbeatRecv" : ISODate("2023-05-24T13:44:08.131Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "c2-mongoshdb02:27017",
"syncSourceId" : 1,
"infoMessage" : "",
"configVersion" : 6,
"configTerm" : 31530
},
{
"_id" : 4,
"name" : "c1-mongoshrd02:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 72838,
"optime" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"optimeDurable" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"optimeDate" : ISODate("2023-05-23T17:30:10Z"),
"optimeDurableDate" : ISODate("2023-05-23T17:30:10Z"),
"lastAppliedWallTime" : ISODate("2023-05-23T17:30:10.839Z"),
"lastDurableWallTime" : ISODate("2023-05-23T17:30:10.839Z"),
"lastHeartbeat" : ISODate("2023-05-24T13:44:08.130Z"),
"lastHeartbeatRecv" : ISODate("2023-05-24T13:44:08.170Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "c2-mongoshdb02:27017",
"syncSourceId" : 1,
"infoMessage" : "",
"configVersion" : 6,
"configTerm" : 31530
},
{
"_id" : 5,
"name" : "c2-mongoshrd03:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 72901,
"optime" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"optimeDurable" : {
"ts" : Timestamp(1684863010, 1),
"t" : NumberLong(31529)
},
"optimeDate" : ISODate("2023-05-23T17:30:10Z"),
"optimeDurableDate" : ISODate("2023-05-23T17:30:10Z"),
"lastAppliedWallTime" : ISODate("2023-05-23T17:30:10.839Z"),
"lastDurableWallTime" : ISODate("2023-05-23T17:30:10.839Z"),
"lastHeartbeat" : ISODate("2023-05-24T13:44:08.074Z"),
"lastHeartbeatRecv" : ISODate("2023-05-24T13:44:08.635Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "c2-mongoshdb02:27017",
"syncSourceId" : 1,
"infoMessage" : "",
"configVersion" : 6,
"configTerm" : 31530
},
{
"_id" : 6,
"name" : "nxt-db-arb:27017",
"health" : 1,
"state" : 7,
"stateStr" : "ARBITER",
"uptime" : 1140988,
"lastHeartbeat" : ISODate("2023-05-24T13:44:08.357Z"),
"lastHeartbeatRecv" : ISODate("2023-05-24T13:44:08.914Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : 6,
"configTerm" : 31530
}
],
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1684863052, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
},
"operationTime" : Timestamp(1684863010, 1)
}