Mongodb 4.4 STARTUP2 state still participate in write majorities.--this is bug?

Ops and Admin Replication
1

from doc:

w: “majority” Behavior
Starting in MongoDB 4.4, replica set members in the STARTUP2 state do not participate in write majorities.

but i test,i find that STARTUP2 state still participate in write majorities.
i’m psa replication.when i add new node to it. the new node state is startup2.
but replication writableVotingMembersCount from 2 to 3.
so i insert data with writeConcern that is hang or timeout.

the follwoing is my test:
shard1:PRIMARY> rs.status();
{
“set” : “shard1”,
“date” : ISODate(“2022-11-21T05:02:51.701Z”),
“myState” : 1,
“term” : NumberLong(2),
“syncSourceHost” : “”,
“syncSourceId” : -1,
“heartbeatIntervalMillis” : NumberLong(2000),
“majorityVoteCount” : 2,
“writeMajorityCount” : 2,
“votingMembersCount” : 3,
“writableVotingMembersCount” : 2,

    "members" : [
            {
                    "_id" : 0,
                    "name" : "100.130.10.149:41001",
                    "health" : 1,
                    "state" : 2,
                    "stateStr" : "SECONDARY",
                    "uptime" : 200,
                    "optime" : {
                            "ts" : Timestamp(1669006964, 1),
                            "t" : NumberLong(2)
                    },
                    "optimeDurable" : {
                            "ts" : Timestamp(1669006964, 1),
                            "t" : NumberLong(2)
                    },
                    "optimeDate" : ISODate("2022-11-21T05:02:44Z"),
                    "optimeDurableDate" : ISODate("2022-11-21T05:02:44Z"),
                    "lastHeartbeat" : ISODate("2022-11-21T05:02:50.799Z"),
                    "lastHeartbeatRecv" : ISODate("2022-11-21T05:02:49.782Z"),
                    "pingMs" : NumberLong(0),
                    "lastHeartbeatMessage" : "",
                    "syncSourceHost" : "100.130.9.150:41001",
                    "syncSourceId" : 1,
                    "infoMessage" : "",
                    "configVersion" : 1,
                    "configTerm" : 2
            },
            {
                    "_id" : 1,
                    "name" : "100.130.9.150:41001",
                    "health" : 1,
                    "state" : 1,
                    "stateStr" : "PRIMARY",
                    "uptime" : 5797,
                    "optime" : {
                            "ts" : Timestamp(1669006964, 1),
                            "t" : NumberLong(2)
                    },
                    "optimeDate" : ISODate("2022-11-21T05:02:44Z"),
                    "syncSourceHost" : "",
                    "syncSourceId" : -1,
                    "infoMessage" : "",
                    "electionTime" : Timestamp(1669005454, 1),
                    "electionDate" : ISODate("2022-11-21T04:37:34Z"),
                    "configVersion" : 1,
                    "configTerm" : 2,
                    "self" : true,
                    "lastHeartbeatMessage" : ""
            },
            {
                    "_id" : 2,
                    "name" : "100.130.10.150:41001",
                    "health" : 1,
                    "state" : 7,
                    "stateStr" : "ARBITER",
                    "uptime" : 5399,
                    "lastHeartbeat" : ISODate("2022-11-21T05:02:50.151Z"),
                    "lastHeartbeatRecv" : ISODate("2022-11-21T05:02:50.159Z"),
                    "pingMs" : NumberLong(0),
                    "lastHeartbeatMessage" : "",
                    "syncSourceHost" : "",
                    "syncSourceId" : -1,
                    "infoMessage" : "",
                    "configVersion" : 1,
                    "configTerm" : 2
            }
    ],
    "ok" : 1,
    "$clusterTime" : {
            "clusterTime" : Timestamp(1669006964, 1),
            "signature" : {
                    "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                    "keyId" : NumberLong(0)
            }
    },
    "operationTime" : Timestamp(1669006964, 1)

}

shard1:PRIMARY> rs.printSlaveReplicationInfo()
source: 100.130.10.149:41001
syncedTo: Mon Nov 21 2022 13:03:54 GMT+0800 (CST)
0 secs (0 hrs) behind the primary

shard1:PRIMARY> db.POCCOLL.insert({_id:1,name:“testWriteConcern”},{writeConcern:{w:“majority”,wtimeout:5000}})
WriteResult({ “nInserted” : 1 })

shard1:PRIMARY> rs.add(“100.130.9.149:41001”)
{
“ok” : 1,
“$clusterTime” : {
“clusterTime” : Timestamp(1669007243, 1),
“signature” : {
“hash” : BinData(0,“AAAAAAAAAAAAAAAAAAAAAAAAAAA=”),
“keyId” : NumberLong(0)
}
},
“operationTime” : Timestamp(1669007243, 1)
}
shard1:PRIMARY> rs.printSlaveReplicationInfo()
source: 100.130.10.149:41001
syncedTo: Mon Nov 21 2022 13:07:23 GMT+0800 (CST)
0 secs (0 hrs) behind the primary
source: 100.130.9.149:41001
syncedTo: Thu Jan 01 1970 08:00:00 GMT+0800 (CST)
1669007243 secs (463613.12 hrs) behind the primary

shard1:PRIMARY> rs.status();
{
“set” : “shard1”,
“date” : ISODate(“2022-11-21T05:07:36.186Z”),
“myState” : 1,
“term” : NumberLong(2),
“syncSourceHost” : “”,
“syncSourceId” : -1,
“heartbeatIntervalMillis” : NumberLong(2000),
“majorityVoteCount” : 3,
“writeMajorityCount” : 3,
“votingMembersCount” : 4,
“writableVotingMembersCount” : 3,
“members” : [
{
“_id” : 0,
“name” : “100.130.10.149:41001”,
“health” : 1,
“state” : 2,
“stateStr” : “SECONDARY”,
“uptime” : 485,
“optime” : {
“ts” : Timestamp(1669007254, 1),
“t” : NumberLong(2)
},
“optimeDurable” : {
“ts” : Timestamp(1669007254, 1),
“t” : NumberLong(2)
},
“optimeDate” : ISODate(“2022-11-21T05:07:34Z”),
“optimeDurableDate” : ISODate(“2022-11-21T05:07:34Z”),
“lastHeartbeat” : ISODate(“2022-11-21T05:07:35.802Z”),
“lastHeartbeatRecv” : ISODate(“2022-11-21T05:07:35.807Z”),
“pingMs” : NumberLong(0),
“lastHeartbeatMessage” : “”,
“syncSourceHost” : “100.130.9.150:41001”,
“syncSourceId” : 1,
“infoMessage” : “”,
“configVersion” : 2,
“configTerm” : 2
},
{
“_id” : 1,
“name” : “100.130.9.150:41001”,
“health” : 1,
“state” : 1,
“stateStr” : “PRIMARY”,
“uptime” : 6082,
“optime” : {
“ts” : Timestamp(1669007254, 1),
“t” : NumberLong(2)
},
“optimeDate” : ISODate(“2022-11-21T05:07:34Z”),
“syncSourceHost” : “”,
“syncSourceId” : -1,
“infoMessage” : “”,
“electionTime” : Timestamp(1669005454, 1),
“electionDate” : ISODate(“2022-11-21T04:37:34Z”),
“configVersion” : 2,
“configTerm” : 2,
“self” : true,
“lastHeartbeatMessage” : “”
},
{
“_id” : 2,
“name” : “100.130.10.150:41001”,
“health” : 1,
“state” : 7,
“stateStr” : “ARBITER”,
“uptime” : 5684,
“lastHeartbeat” : ISODate(“2022-11-21T05:07:35.802Z”),
“lastHeartbeatRecv” : ISODate(“2022-11-21T05:07:35.805Z”),
“pingMs” : NumberLong(0),
“lastHeartbeatMessage” : “”,
“syncSourceHost” : “”,
“syncSourceId” : -1,
“infoMessage” : “”,
“configVersion” : 2,
“configTerm” : 2
},
{
“_id” : 3,
“name” : “100.130.9.149:41001”,
“health” : 1,
“state” : 5,
“stateStr” : “STARTUP2”,
“uptime” : 12,
“optime” : {
“ts” : Timestamp(0, 0),
“t” : NumberLong(-1)
},
“optimeDurable” : {
“ts” : Timestamp(0, 0),
“t” : NumberLong(-1)
},
“optimeDate” : ISODate(“1970-01-01T00:00:00Z”),
“optimeDurableDate” : ISODate(“1970-01-01T00:00:00Z”),
“lastHeartbeat” : ISODate(“2022-11-21T05:07:35.813Z”),
“lastHeartbeatRecv” : ISODate(“2022-11-21T05:07:35.327Z”),
“pingMs” : NumberLong(0),
“lastHeartbeatMessage” : “”,
“syncSourceHost” : “100.130.9.150:41001”,
“syncSourceId” : 1,
“infoMessage” : “”,
“configVersion” : 2,
“configTerm” : 2
}
],
}

so now i insert data again ,startup2 state participate in write majorities.
so it is bug for it or other reason?

shard1:PRIMARY> db.POCCOLL.insert({_id:3,name:“testWriteConcern”},{writeConcern:{w:“majority”,wtimeout:5000}})
WriteResult({
“nInserted” : 1,
“writeConcernError” : {
“code” : 64,
“codeName” : “WriteConcernFailed”,
“errmsg” : “waiting for replication timed out”,
“errInfo” : {
“wtimeout” : true,
“writeConcern” : {
“w” : “majority”,
“wtimeout” : 5000,
“provenance” : “clientSupplied”
}
}
}
})

https://jira.mongodb.org/browse/SERVER-71509

2

Hi @jing_xu,

Arbiters are evil. I have already explained this at length in multiple posts in this forum that you can dig out.

From what I understand, you now have 4 nodes in your RS including one Arbiter which cannot acknowledge write operations but DO count as a voting member of the RS.

To be able to write with w=majority, by definition, you need to be able to write to the majority of the voting members in this RS.

With these 4 nodes, your majority is now at 3. (4 voting nodes / 2 +1 = 3 nodes). With the Arbiter you already lost your only chance to have a node down. And because your 4th node is still in STARTUP2 (initial sync I guess) you cannot write to this RS with majority > 2 as 2 of your nodes cannot acknowledge write operations at the moment.

Replace the Arbiter with a normal and it will work just fine.

Also don’t stay at 4 nodes. It’s either 3 or 5 but 4 isn’t optimal as your majority is at 3 with 4 nodes meaning that you can only afford to lose a single node (just like when you only have 3 nodes). With 5 nodes you can lose up to 2 machines as your majority is still at 3 but you have 5 nodes now.

To sum up this once for all: You CANNOT use w=majority if you are running an arbiter. Else your RS is not Highly Available (HA) which is alarming because Replica Sets exist for this very reason: make your cluster HA.

PSA => Shouldn’t try to write with more than W=1 (while majority stands at 2).
PSSSA => W=2 maximum (while majority stands at 3).

And this is already a trade off because you are trading HA vs 1 arbiter (=less cost). Atlas doesn’t propose arbiters in the configurations to keep things simple and HA.

Cheers,
Maxime.

3

hi Maxime:
i want to ask about:
Starting in MongoDB 4.4, replica set members in the STARTUP2 state do not participate in write majorities
from https://www.mongodb.com/docs/v4.4/reference/write-concern/#causally-consistent-sessions-and-write-concerns.

i think it startup2 state do not participate in write majorities for replication. but i add startup2 state node to PSA. so it is still participate in write majorities.

I want to say that this official document is not accurate about Starting in MongoDB 4.4, replica set members in the STARTUP2 state do not participate in write majorities.

4

Hi @jing_xu,

And if you read just below your section in the next one:

They explain what I was explaining above.

STARTUP2 node do NOT participate in write majorities => They do not acknowledge write operations. You have to rely on other data-bearing nodes to do so as STARTUP2 nodes are still catching up with the cluster.

BUT they do count as take part in the cluster and its configuration - including the calculation of the majority the write concern.

The lesson here is that you shouldn’t use w=“majority” with a P-S-A cluster because if P or S fail, you can’t write anymore. You need a P-S-S for this so you can at least lose one node and keep everything running. Else you are not Highly Available and almost all the nodes are SPOF. With P-S-A it’s w=1 maximum to be somewhat HA.

5

hi Maxime:
This is no different from the 4.2 and 4.4, it’s all participate write majorities.so I understand that 4.4 startup2 not participate in write majorities. It will not participate in the statistics until the status becomes secondary.

writeConcern
writeConcern997×640 42.2 KB

6

My understanding (meaning I could be wrong) is that in 4.2 and down, RS members in STARTUP2 were able to participate in write majorities (=they were able to acknowledge a write operation with w=majority).

This doesn’t mean that STARTUP2 members don’t take part in the votes as they are part of the configuration.

That being said, apparently there is an exception for “newly added” in the RS. Which makes sense in your case but I don’t know when this “newly added” status stops…

7

hi Maxime:
i test 4.4 doc it is wrong. 5.0 startup2 node do NOT participate in write majorities 4.4 still
do participate in write majorities.so i think it is wrong.
Starting in MongoDB 4.4, replica set members in the STARTUP2 state do not participate in write majorities
from https://www.mongodb.com/docs/v4.4/reference/write-concern/#causally-consistent-sessions-and-write-concerns
shard1 [direct: primary] test> db.version()
4.4.18

shard1 [direct: primary] test> rs.status()
{
set: ‘shard1’,
date: ISODate(“2022-12-09T08:26:29.625Z”),
myState: 1,
term: Long(“1”),
syncSourceHost: ‘’,
syncSourceId: -1,
heartbeatIntervalMillis: Long(“2000”),
majorityVoteCount: 2,
writeMajorityCount: 2,
votingMembersCount: 2,
writableVotingMembersCount: 2,
optimes: {
lastCommittedOpTime: { ts: Timestamp({ t: 1670574382, i: 3656 }), t: Long(“1”) },
lastCommittedWallTime: ISODate(“2022-12-09T08:26:22.030Z”),
readConcernMajorityOpTime: { ts: Timestamp({ t: 1670574382, i: 3656 }), t: Long(“1”) },
readConcernMajorityWallTime: ISODate(“2022-12-09T08:26:22.030Z”),
appliedOpTime: { ts: Timestamp({ t: 1670574389, i: 29720 }), t: Long(“1”) },
durableOpTime: { ts: Timestamp({ t: 1670574389, i: 29720 }), t: Long(“1”) },
lastAppliedWallTime: ISODate(“2022-12-09T08:26:29.200Z”),
lastDurableWallTime: ISODate(“2022-12-09T08:26:29.200Z”)
},
lastStableRecoveryTimestamp: Timestamp({ t: 1670574339, i: 56832 }),
electionCandidateMetrics: {
lastElectionReason: ‘electionTimeout’,
lastElectionDate: ISODate(“2022-12-09T08:22:36.812Z”),
electionTerm: Long(“1”),
lastCommittedOpTimeAtElection: { ts: Timestamp({ t: 0, i: 0 }), t: Long(“-1”) },
lastSeenOpTimeAtElection: { ts: Timestamp({ t: 1670574156, i: 1 }), t: Long(“-1”) },
numVotesNeeded: 1,
priorityAtElection: 1,
electionTimeoutMillis: Long(“10000”),
newTermStartDate: ISODate(“2022-12-09T08:22:36.839Z”),
wMajorityWriteAvailabilityDate: ISODate(“2022-12-09T08:22:36.862Z”)
},
members: [
{
_id: 0,
name: ‘10.130.10.149:41001’,
health: 1,
state: 1,
stateStr: ‘PRIMARY’,
uptime: 261,
optime: { ts: Timestamp({ t: 1670574389, i: 29720 }), t: Long(“1”) },
optimeDate: ISODate(“2022-12-09T08:26:29.000Z”),
lastAppliedWallTime: ISODate(“2022-12-09T08:26:29.200Z”),
lastDurableWallTime: ISODate(“2022-12-09T08:26:29.200Z”),
syncSourceHost: ‘’,
syncSourceId: -1,
infoMessage: ‘’,
electionTime: Timestamp({ t: 1670574156, i: 2 }),
electionDate: ISODate(“2022-12-09T08:22:36.000Z”),
configVersion: 2,
configTerm: 1,
self: true,
lastHeartbeatMessage: ‘’
},
{
_id: 1,
name: ‘10.130.9.149:41001’,
health: 1,
state: 5,
stateStr: ‘STARTUP2’,
uptime: 7,
optime: { ts: Timestamp({ t: 0, i: 0 }), t: Long(“-1”) },
optimeDurable: { ts: Timestamp({ t: 0, i: 0 }), t: Long(“-1”) },
optimeDate: ISODate(“1970-01-01T00:00:00.000Z”),
optimeDurableDate: ISODate(“1970-01-01T00:00:00.000Z”),
lastAppliedWallTime: ISODate(“1970-01-01T00:00:00.000Z”),
lastDurableWallTime: ISODate(“1970-01-01T00:00:00.000Z”),
lastHeartbeat: ISODate(“2022-12-09T08:26:28.090Z”),
lastHeartbeatRecv: ISODate(“2022-12-09T08:26:27.711Z”),
pingMs: Long(“5”),
lastHeartbeatMessage: ‘’,
syncSourceHost: ‘10.130.10.149:41001’,
syncSourceId: 0,
infoMessage: ‘’,
configVersion: 2,
configTerm: 1
}
],
ok: 1,
‘$clusterTime’: {
clusterTime: Timestamp({ t: 1670574389, i: 29720 }),
signature: {
hash: Binary(Buffer.from(“0000000000000000000000000000000000000000”, “hex”), 0),
keyId: Long(“0”)
}
},
operationTime: Timestamp({ t: 1670574389, i: 29720 })
}

shard1 [direct: primary] test> db.version()
5.0.3
shard1 [direct: primary] test> rs.status()
{
set: ‘shard1’,
date: ISODate(“2022-12-09T08:14:07.209Z”),
myState: 1,
term: Long(“1”),
syncSourceHost: ‘’,
syncSourceId: -1,
heartbeatIntervalMillis: Long(“2000”),
majorityVoteCount: 1,
writeMajorityCount: 1,
votingMembersCount: 1,
writableVotingMembersCount: 1,
optimes: {
lastCommittedOpTime: { ts: Timestamp({ t: 1670573647, i: 19992 }), t: Long(“1”) },
lastCommittedWallTime: ISODate(“2022-12-09T08:14:07.168Z”),
readConcernMajorityOpTime: { ts: Timestamp({ t: 1670573647, i: 19992 }), t: Long(“1”) },
appliedOpTime: { ts: Timestamp({ t: 1670573647, i: 23064 }), t: Long(“1”) },
durableOpTime: { ts: Timestamp({ t: 1670573647, i: 19992 }), t: Long(“1”) },
lastAppliedWallTime: ISODate(“2022-12-09T08:14:07.202Z”),
lastDurableWallTime: ISODate(“2022-12-09T08:14:07.168Z”)
},
lastStableRecoveryTimestamp: Timestamp({ t: 1670573623, i: 45092 }),
electionCandidateMetrics: {
lastElectionReason: ‘electionTimeout’,
lastElectionDate: ISODate(“2022-12-09T08:08:38.675Z”),
electionTerm: Long(“1”),
lastCommittedOpTimeAtElection: { ts: Timestamp({ t: 0, i: 0 }), t: Long(“-1”) },
lastSeenOpTimeAtElection: { ts: Timestamp({ t: 1670573318, i: 1 }), t: Long(“-1”) },
numVotesNeeded: 1,
priorityAtElection: 1,
electionTimeoutMillis: Long(“10000”),
newTermStartDate: ISODate(“2022-12-09T08:08:38.691Z”),
wMajorityWriteAvailabilityDate: ISODate(“2022-12-09T08:08:38.701Z”)
},
members: [
{
_id: 0,
name: ‘10.130.10.149:51001’,
health: 1,
state: 1,
stateStr: ‘PRIMARY’,
uptime: 336,
optime: { ts: Timestamp({ t: 1670573647, i: 23064 }), t: Long(“1”) },
optimeDate: ISODate(“2022-12-09T08:14:07.000Z”),
syncSourceHost: ‘’,
syncSourceId: -1,
infoMessage: ‘’,
electionTime: Timestamp({ t: 1670573318, i: 2 }),
electionDate: ISODate(“2022-12-09T08:08:38.000Z”),
configVersion: 2,
configTerm: 1,
self: true,
lastHeartbeatMessage: ‘’
},
{
_id: 1,
name: ‘10.130.9.149:51001’,
health: 1,
state: 5,
stateStr: ‘STARTUP2’,
uptime: 6,
optime: { ts: Timestamp({ t: 0, i: 0 }), t: Long(“-1”) },
optimeDurable: { ts: Timestamp({ t: 0, i: 0 }), t: Long(“-1”) },
optimeDate: ISODate(“1970-01-01T00:00:00.000Z”),
optimeDurableDate: ISODate(“1970-01-01T00:00:00.000Z”),
lastHeartbeat: ISODate(“2022-12-09T08:14:06.367Z”),
lastHeartbeatRecv: ISODate(“2022-12-09T08:14:06.392Z”),
pingMs: Long(“3”),
lastHeartbeatMessage: ‘’,
syncSourceHost: ‘10.130.10.149:51001’,
syncSourceId: 0,
infoMessage: ‘’,
configVersion: 2,
configTerm: 1
}
],
ok: 1,
‘$clusterTime’: {
clusterTime: Timestamp({ t: 1670573647, i: 23064 }),
signature: {
hash: Binary(Buffer.from(“0000000000000000000000000000000000000000”, “hex”), 0),
keyId: Long(“0”)
}
},
operationTime: Timestamp({ t: 1670573647, i: 23064 })
}