Upgrade RS to MongoDB v5 issue

Ops and Admin Replication
1

Good day all !

We have been trying to upgrade our local 3-node Replica Set from v4.4 to v5 , took down one of the secondary instances and replaced the 4.4 binaries with 5.0 binaries (RHEL 7).

When starting the instance with the new binaries we get the below error message and instance fails to start:

{"t":{"$date":"2023-11-14T09:35:00.801+02:00"},"s":"F",  "c":"-",        "id":23095,   "ctx":"ReplCoord-0","msg":"Fatal assertion","attr":{"msgid":5624100,"error":"IllegalOperation:###","file":"src/mongo/db/repl/repl_set_config_checks.cpp","line":444}}
{"t":{"$date":"2023-11-14T09:35:00.801+02:00"},"s":"I",  "c":"STORAGE",  "id":20320,   "ctx":"initandlisten","msg":"createCollection","attr":{"namespace":"local.system.views","uuidDisposition":"generated","uuid":{"uuid":{"$uuid":"2dc436de-9699-47c1-9957-385be320e866"}},"options":{}}}
{"t":{"$date":"2023-11-14T09:35:00.802+02:00"},"s":"F",  "c":"-",        "id":23096,   "ctx":"ReplCoord-0","msg":"\n\n***aborting after fassert() failure\n\n"}

Has anyone faced similar behavior or any ideas on how to resolve this issue?

Thank you in advance!

2

Hi @GeorgeK79

What does the rs.status() look like? Redact as appropriate.

3

Hi Chris,

RS status looks healthy, below the full output:

uatrs0:PRIMARY> rs.status()
{
        "set" : "uatrs0",
        "date" : ISODate("2023-11-15T06:09:57.780Z"),
        "myState" : 1,
        "term" : NumberLong(152),
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "heartbeatIntervalMillis" : NumberLong(2000),
        "majorityVoteCount" : 2,
        "writeMajorityCount" : 2,
        "votingMembersCount" : 3,
        "writableVotingMembersCount" : 3,
        "optimes" : {
                "lastCommittedOpTime" : {
                        "ts" : Timestamp(1700028597, 1),
                        "t" : NumberLong(152)
                },
                "lastCommittedWallTime" : ISODate("2023-11-15T06:09:57.659Z"),
                "readConcernMajorityOpTime" : {
                        "ts" : Timestamp(1700028597, 1),
                        "t" : NumberLong(152)
                },
                "readConcernMajorityWallTime" : ISODate("2023-11-15T06:09:57.659Z"),
                "appliedOpTime" : {
                        "ts" : Timestamp(1700028597, 2),
                        "t" : NumberLong(152)
                },
                "durableOpTime" : {
                        "ts" : Timestamp(1700028597, 1),
                        "t" : NumberLong(152)
                },
                "lastAppliedWallTime" : ISODate("2023-11-15T06:09:57.779Z"),
                "lastDurableWallTime" : ISODate("2023-11-15T06:09:57.659Z")
        },
        "lastStableRecoveryTimestamp" : Timestamp(1700028580, 47),
        "electionCandidateMetrics" : {
                "lastElectionReason" : "electionTimeout",
                "lastElectionDate" : ISODate("2023-08-22T01:56:19.011Z"),
                "electionTerm" : NumberLong(152),
                "lastCommittedOpTimeAtElection" : {
                        "ts" : Timestamp(1692669241, 2),
                        "t" : NumberLong(151)
                },
                "lastSeenOpTimeAtElection" : {
                        "ts" : Timestamp(1692669335, 16),
                        "t" : NumberLong(151)
                },
                "numVotesNeeded" : 2,
                "priorityAtElection" : 1,
                "electionTimeoutMillis" : NumberLong(10000),
                "numCatchUpOps" : NumberLong(0),
                "newTermStartDate" : ISODate("2023-08-22T01:56:19.472Z"),
                "wMajorityWriteAvailabilityDate" : ISODate("2023-08-22T01:56:20.185Z")
        },
        "members" : [
                {
                        "_id" : 0,
                        "name" : "XXXXXXXut1.domain:27021",
                        "health" : 1,
                        "state" : 1,
                        "stateStr" : "PRIMARY",
                        "uptime" : 10790725,
                        "optime" : {
                                "ts" : Timestamp(1700028597, 2),
                                "t" : NumberLong(152)
                        },
                        "optimeDate" : ISODate("2023-11-15T06:09:57Z"),
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "electionTime" : Timestamp(1692669379, 1),
                        "electionDate" : ISODate("2023-08-22T01:56:19Z"),
                        "configVersion" : 24,
                        "configTerm" : 152,
                        "self" : true,
                        "lastHeartbeatMessage" : ""
                },
                {
                        "_id" : 1,
                        "name" : "XXXXXXut2.domain:27021",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 79519,
                        "optime" : {
                                "ts" : Timestamp(1700028595, 16),
                                "t" : NumberLong(152)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1700028595, 16),
                                "t" : NumberLong(152)
                        },
                        "optimeDate" : ISODate("2023-11-15T06:09:55Z"),
                        "optimeDurableDate" : ISODate("2023-11-15T06:09:55Z"),
                        "lastHeartbeat" : ISODate("2023-11-15T06:09:56.280Z"),
                        "lastHeartbeatRecv" : ISODate("2023-11-15T06:09:56.348Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "XXXXXXXut1.domain:27021",
                        "syncSourceId" : 0,
                        "infoMessage" : "",
                        "configVersion" : 24,
                        "configTerm" : 152
                },
                {
                        "_id" : 2,
                        "name" : "XXXXXXut3.domain:27021",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 66861,
                        "optime" : {
                                "ts" : Timestamp(1700028595, 16),
                                "t" : NumberLong(152)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1700028595, 16),
                                "t" : NumberLong(152)
                        },
                        "optimeDate" : ISODate("2023-11-15T06:09:55Z"),
                        "optimeDurableDate" : ISODate("2023-11-15T06:09:55Z"),
                        "lastHeartbeat" : ISODate("2023-11-15T06:09:57.612Z"),
                        "lastHeartbeatRecv" : ISODate("2023-11-15T06:09:57.636Z"),
                        "pingMs" : NumberLong(2),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "XXXXXXut2.domain:27021",
                        "syncSourceId" : 1,
                        "infoMessage" : "",
                        "configVersion" : 24,
                        "configTerm" : 152
                }
        ],
        "ok" : 1,
        "$clusterTime" : {
                "clusterTime" : Timestamp(1700028597, 2),
                "signature" : {
                        "hash" : BinData(0,"CyXj32eZxyS40w7dVovFHQOt1f0="),
                        "keyId" : NumberLong("7246678373265571856")
                }
        },
        "operationTime" : Timestamp(1700028597, 2)
}
4

Sorry @GeorgeK79 , I meant to ask for the rs.conf() muscle memory made me type rs.status()

I’m looking for customised getLastErrorDefaults , these conflict with 5.0 and should be set with setDefaultRWConcern

5

Hi @chris ,

Indeed that was the culprit, we had a custom set ‘getLastErrorDefaults’ :

"getLastErrorDefaults" : {
 "w : "majority",
 "wtimeout" : 3600
                }

When I changed it to the below, the upgrade to MongoDB v5.0 has been completed smoothly without any errors:

cfg = rs.conf()
cfg.settings.getLastErrorDefaults.w = "1"
cfg.settings.getLastErrorDefaults.wtimeout = 0
rs.reconfig(cfg)

Many thanks for your support on this!

1 Like
6

This topic was automatically closed 5 days after the last reply. New replies are no longer allowed.