I have facing Write Operation issue on MongoDB Primary node

[Sanjay_Soni]

I have 4 MongoDB servers and One Arbiter node. 4 out of 2 servers are down then remaining 2 are working as primary and secondary. At that time i supposed to run write operation on the Primary Node then it stuck And I have to re-login on server I can fetch a new data which was I push through in insert statement.

mongocluster:PRIMARY> rs.status()
{
“set” : “mongocluster”,
“date” : ISODate(“2022-11-08T07:09:48.980Z”),
“myState” : 1,
“term” : NumberLong(74),
“syncSourceHost” : “”,
“syncSourceId” : -1,
“heartbeatIntervalMillis” : NumberLong(2000),
“majorityVoteCount” : 3,
“writeMajorityCount” : 3,
“votingMembersCount” : 5,
“writableVotingMembersCount” : 4,
“optimes” : {
“lastCommittedOpTime” : {
“ts” : Timestamp(1667887148, 2),
“t” : NumberLong(67)
},
“lastCommittedWallTime” : ISODate(“2022-11-08T05:59:08.814Z”),
“readConcernMajorityOpTime” : {
“ts” : Timestamp(1667887148, 2),
“t” : NumberLong(67)
},
“appliedOpTime” : {
“ts” : Timestamp(1667891386, 1),
“t” : NumberLong(74)
},
“durableOpTime” : {
“ts” : Timestamp(1667891386, 1),
“t” : NumberLong(74)
},
“lastAppliedWallTime” : ISODate(“2022-11-08T07:09:46.694Z”),
“lastDurableWallTime” : ISODate(“2022-11-08T07:09:46.694Z”)
},
“lastStableRecoveryTimestamp” : Timestamp(1667887148, 2),
“electionCandidateMetrics” : {
“lastElectionReason” : “electionTimeout”,
“lastElectionDate” : ISODate(“2022-11-08T07:06:46.673Z”),
“electionTerm” : NumberLong(74),
“lastCommittedOpTimeAtElection” : {
“ts” : Timestamp(1667887148, 2),
“t” : NumberLong(67)
},
“lastSeenOpTimeAtElection” : {
“ts” : Timestamp(1667891193, 1),
“t” : NumberLong(73)
},
“numVotesNeeded” : 3,
“priorityAtElection” : 1,
“electionTimeoutMillis” : NumberLong(10000),
“numCatchUpOps” : NumberLong(0),
“newTermStartDate” : ISODate(“2022-11-08T07:06:46.689Z”)
},
“electionParticipantMetrics” : {
“votedForCandidate” : true,
“electionTerm” : NumberLong(71),
“lastVoteDate” : ISODate(“2022-11-08T07:01:40.459Z”),
“electionCandidateMemberId” : 4,
“voteReason” : “”,
“lastAppliedOpTimeAtElection” : {
“ts” : Timestamp(1667890885, 1),
“t” : NumberLong(70)
},
“maxAppliedOpTimeInSet” : {
“ts” : Timestamp(1667890885, 1),
“t” : NumberLong(70)
},
“priorityAtElection” : 1
},
“members” : [
{
“_id” : 0,
“name” : “mongoDB1:27017”,
“health” : 0,
“state” : 8,
“stateStr” : “(not reachable/healthy)”,
“uptime” : 0,
“optime” : {
“ts” : Timestamp(0, 0),
“t” : NumberLong(-1)
},
“optimeDurable” : {
“ts” : Timestamp(0, 0),
“t” : NumberLong(-1)
},
“optimeDate” : ISODate(“1970-01-01T00:00:00Z”),
“optimeDurableDate” : ISODate(“1970-01-01T00:00:00Z”),
“lastAppliedWallTime” : ISODate(“1970-01-01T00:00:00Z”),
“lastDurableWallTime” : ISODate(“1970-01-01T00:00:00Z”),
“lastHeartbeat” : ISODate(“2022-11-08T07:09:47.032Z”),
“lastHeartbeatRecv” : ISODate(“1970-01-01T00:00:00Z”),
“pingMs” : NumberLong(0),
“lastHeartbeatMessage” : “Error connecting to mongoDB1:27017 (10.0.0.8:27017) :: caused by :: Connection refused”,
“syncSourceHost” : “”,
“syncSourceId” : -1,
“infoMessage” : “”,
“configVersion” : -1,
“configTerm” : -1
},
{
“_id” : 1,
“name” : “mongoDB2:27017”,
“health” : 0,
“state” : 8,
“stateStr” : “(not reachable/healthy)”,
“uptime” : 0,
“optime” : {
“ts” : Timestamp(0, 0),
“t” : NumberLong(-1)
},
“optimeDurable” : {
“ts” : Timestamp(0, 0),
“t” : NumberLong(-1)
},
“optimeDate” : ISODate(“1970-01-01T00:00:00Z”),
“optimeDurableDate” : ISODate(“1970-01-01T00:00:00Z”),
“lastAppliedWallTime” : ISODate(“1970-01-01T00:00:00Z”),
“lastDurableWallTime” : ISODate(“1970-01-01T00:00:00Z”),
“lastHeartbeat” : ISODate(“2022-11-08T07:09:48.907Z”),
“lastHeartbeatRecv” : ISODate(“1970-01-01T00:00:00Z”),
“pingMs” : NumberLong(0),
“lastHeartbeatMessage” : “Error connecting to mongoDB2:27017 (10.0.0.11:27017) :: caused by :: Connection refused”,
“syncSourceHost” : “”,
“syncSourceId” : -1,
“infoMessage” : “”,
“configVersion” : -1,
“configTerm” : -1
},
{
“_id” : 3,
“name” : “mongoDB4:27017”,
“health” : 1,
“state” : 1,
“stateStr” : “PRIMARY”,
“uptime” : 497,
“optime” : {
“ts” : Timestamp(1667891386, 1),
“t” : NumberLong(74)
},
“optimeDate” : ISODate(“2022-11-08T07:09:46Z”),
“lastAppliedWallTime” : ISODate(“2022-11-08T07:09:46.694Z”),
“lastDurableWallTime” : ISODate(“2022-11-08T07:09:46.694Z”),
“syncSourceHost” : “”,
“syncSourceId” : -1,
“infoMessage” : “”,
“electionTime” : Timestamp(1667891206, 1),
“electionDate” : ISODate(“2022-11-08T07:06:46Z”),
“configVersion” : 35,
“configTerm” : 74,
“self” : true,
“lastHeartbeatMessage” : “”
},
{
“_id” : 4,
“name” : “mongoDB3:27017”,
“health” : 1,
“state” : 2,
“stateStr” : “SECONDARY”,
“uptime” : 184,
“optime” : {
“ts” : Timestamp(1667891386, 1),
“t” : NumberLong(74)
},
“optimeDurable” : {
“ts” : Timestamp(1667891386, 1),
“t” : NumberLong(74)
},
“optimeDate” : ISODate(“2022-11-08T07:09:46Z”),
“optimeDurableDate” : ISODate(“2022-11-08T07:09:46Z”),
“lastAppliedWallTime” : ISODate(“2022-11-08T07:09:46.694Z”),
“lastDurableWallTime” : ISODate(“2022-11-08T07:09:46.694Z”),
“lastHeartbeat” : ISODate(“2022-11-08T07:09:48.714Z”),
“lastHeartbeatRecv” : ISODate(“2022-11-08T07:09:47.728Z”),
“pingMs” : NumberLong(0),
“lastHeartbeatMessage” : “”,
“syncSourceHost” : “mongoDB4:27017”,
“syncSourceId” : 3,
“infoMessage” : “”,
“configVersion” : 35,
“configTerm” : 74
},
{
“_id” : 5,
“name” : “Citus1:27017”,
“health” : 1,
“state” : 7,
“stateStr” : “ARBITER”,
“uptime” : 495,
“lastHeartbeat” : ISODate(“2022-11-08T07:09:48.387Z”),
“lastHeartbeatRecv” : ISODate(“2022-11-08T07:09:48.387Z”),
“pingMs” : NumberLong(34),
“lastHeartbeatMessage” : “”,
“syncSourceHost” : “”,
“syncSourceId” : -1,
“infoMessage” : “”,
“configVersion” : 35,
“configTerm” : 74
}
],
“ok” : 1,
“$clusterTime” : {
“clusterTime” : Timestamp(1667891386, 1),
“signature” : {
“hash” : BinData(0,“l/B6dJM0LuraYaFRSSH9UwHg3eY=”),
“keyId” : NumberLong(“7146254070720757764”)
}
},
“operationTime” : Timestamp(1667891386, 1)
}
mongocluster:PRIMARY>

mongocluster:PRIMARY>
mongocluster:PRIMARY> rs.conf()
{
“_id” : “mongocluster”,
“version” : 35,
“term” : 76,
“members” : [
{
“_id” : 0,
“host” : “mongoDB1:27017”,
“arbiterOnly” : false,
“buildIndexes” : true,
“hidden” : false,
“priority” : 3,
“tags” : {

		},
		"secondaryDelaySecs" : NumberLong(0),
		"votes" : 1
	},
	{
		"_id" : 1,
		"host" : "mongoDB2:27017",
		"arbiterOnly" : false,
		"buildIndexes" : true,
		"hidden" : false,
		"priority" : 2,
		"tags" : {
			
		},
		"secondaryDelaySecs" : NumberLong(0),
		"votes" : 1
	},
	{
		"_id" : 3,
		"host" : "mongoDB4:27017",
		"arbiterOnly" : false,
		"buildIndexes" : true,
		"hidden" : false,
		"priority" : 1,
		"tags" : {
			
		},
		"secondaryDelaySecs" : NumberLong(0),
		"votes" : 1
	},
	{
		"_id" : 4,
		"host" : "mongoDB3:27017",
		"arbiterOnly" : false,
		"buildIndexes" : true,
		"hidden" : false,
		"priority" : 1,
		"tags" : {
			
		},
		"secondaryDelaySecs" : NumberLong(0),
		"votes" : 1
	},
	{
		"_id" : 5,
		"host" : "Citus1:27017",
		"arbiterOnly" : true,
		"buildIndexes" : true,
		"hidden" : false,
		"priority" : 0,
		"tags" : {
			
		},
		"secondaryDelaySecs" : NumberLong(0),
		"votes" : 1
	}
],
"protocolVersion" : NumberLong(1),
"writeConcernMajorityJournalDefault" : true,
"settings" : {
	"chainingAllowed" : true,
	"heartbeatIntervalMillis" : 2000,
	"heartbeatTimeoutSecs" : 10,
	"electionTimeoutMillis" : 10000,
	"catchUpTimeoutMillis" : -1,
	"catchUpTakeoverDelayMillis" : 30000,
	"getLastErrorModes" : {
		
	},
	"getLastErrorDefaults" : {
		"w" : 1,
		"wtimeout" : 0
	},
	"replicaSetId" : ObjectId("632c982ca07b1a8a7df1e397")
}

}
mongocluster:PRIMARY>

Kindly help to resole this issue why primary node not execute write operations.

[Ramachandra_Tummala]

I think your majority data bearing nodes should acknowledge the write but it is not happening so primary is waiting
You have majority but arbiter is non data bearing
Check this thread

Best practices ReplicaSet MongoDB

Good day, colleagues! We use a highly loaded MongoDB database in a ReplicaSet configuration, which includes one primary node, 3 secondary nodes and one arbiter. One of the secondary nodes became in the recovering state, and at the same time, one of the secondary nodes was stopped. Multiple “slow query” type errors were recorded. We are using version 4.4.3. Tell me what configuration we should use to avoid such problems and maybe there are some best practices for configuring the database.