Hi @Kobe_W, Hello @Kushagra_Kesav,
Thank you for your replies.
Before
Go driver successfully connected, no errors. Host in connection string is: solidatus-db-0.db
RS status & config output before
rs0:PRIMARY> rs.status()
{
"set" : "rs0",
"date" : ISODate("2023-03-22T08:30:34.395Z"),
"myState" : 1,
"term" : NumberLong(2),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"majorityVoteCount" : 1,
"writeMajorityCount" : 1,
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1679473830, 7),
"t" : NumberLong(2)
},
"lastCommittedWallTime" : ISODate("2023-03-22T08:30:30.674Z"),
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1679473830, 7),
"t" : NumberLong(2)
},
"readConcernMajorityWallTime" : ISODate("2023-03-22T08:30:30.674Z"),
"appliedOpTime" : {
"ts" : Timestamp(1679473830, 7),
"t" : NumberLong(2)
},
"durableOpTime" : {
"ts" : Timestamp(1679473830, 7),
"t" : NumberLong(2)
},
"lastAppliedWallTime" : ISODate("2023-03-22T08:30:30.674Z"),
"lastDurableWallTime" : ISODate("2023-03-22T08:30:30.674Z")
},
"lastStableRecoveryTimestamp" : Timestamp(1679473810, 1),
"lastStableCheckpointTimestamp" : Timestamp(1679473810, 1),
"electionCandidateMetrics" : {
"lastElectionReason" : "electionTimeout",
"lastElectionDate" : ISODate("2023-01-12T08:13:22.040Z"),
"electionTerm" : NumberLong(2),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1673511115, 1),
"t" : NumberLong(1)
},
"numVotesNeeded" : 1,
"priorityAtElection" : 1,
"electionTimeoutMillis" : NumberLong(10000),
"newTermStartDate" : ISODate("2023-01-12T08:13:22.042Z"),
"wMajorityWriteAvailabilityDate" : ISODate("2023-01-12T08:13:22.094Z")
},
"members" : [
{
"_id" : 1,
"name" : "10.1.95.24:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 5962708,
"optime" : {
"ts" : Timestamp(1679473830, 7),
"t" : NumberLong(2)
},
"optimeDate" : ISODate("2023-03-22T08:30:30Z"),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1673511202, 1),
"electionDate" : ISODate("2023-01-12T08:13:22Z"),
"configVersion" : 182936,
"self" : true,
"lastHeartbeatMessage" : ""
}
],
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1679473830, 7),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
},
"operationTime" : Timestamp(1679473830, 7)
}
rs0:PRIMARY> rs.config()
{
"_id" : "rs0",
"version" : 182936,
"protocolVersion" : NumberLong(1),
"writeConcernMajorityJournalDefault" : true,
"members" : [
{
"_id" : 1,
"host" : "10.1.95.24:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"catchUpTimeoutMillis" : -1,
"catchUpTakeoverDelayMillis" : 30000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
},
"replicaSetId" : ObjectId("6311ccaab8d114d352e0655e")
}
}
Mongo reachable as:
$ date
Wed Mar 22 08:29:50 UTC 2023
$ curl solidatus-db-0.db:27017
It looks like you are trying to access MongoDB over HTTP on the native driver port.
After
Then MongoDB restarts, comes up under a new IP address:
RS status & config output after
rs0:PRIMARY> rs.status()
{
"set" : "rs0",
"date" : ISODate("2023-03-22T08:34:24.906Z"),
"myState" : 1,
"term" : NumberLong(3),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"majorityVoteCount" : 1,
"writeMajorityCount" : 1,
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1679474063, 6),
"t" : NumberLong(3)
},
"lastCommittedWallTime" : ISODate("2023-03-22T08:34:23.612Z"),
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1679474063, 6),
"t" : NumberLong(3)
},
"readConcernMajorityWallTime" : ISODate("2023-03-22T08:34:23.612Z"),
"appliedOpTime" : {
"ts" : Timestamp(1679474063, 6),
"t" : NumberLong(3)
},
"durableOpTime" : {
"ts" : Timestamp(1679474063, 6),
"t" : NumberLong(3)
},
"lastAppliedWallTime" : ISODate("2023-03-22T08:34:23.612Z"),
"lastDurableWallTime" : ISODate("2023-03-22T08:34:23.612Z")
},
"lastStableRecoveryTimestamp" : Timestamp(1679473965, 6),
"lastStableCheckpointTimestamp" : Timestamp(1679473965, 6),
"electionCandidateMetrics" : {
"lastElectionReason" : "electionTimeout",
"lastElectionDate" : ISODate("2023-03-22T08:34:14.203Z"),
"electionTerm" : NumberLong(3),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1679473965, 6),
"t" : NumberLong(2)
},
"numVotesNeeded" : 1,
"priorityAtElection" : 1,
"electionTimeoutMillis" : NumberLong(10000),
"newTermStartDate" : ISODate("2023-03-22T08:34:14.206Z"),
"wMajorityWriteAvailabilityDate" : ISODate("2023-03-22T08:34:14.249Z")
},
"members" : [
{
"_id" : 0,
"name" : "10.1.95.4:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 86,
"optime" : {
"ts" : Timestamp(1679474063, 6),
"t" : NumberLong(3)
},
"optimeDate" : ISODate("2023-03-22T08:34:23Z"),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1679474054, 1),
"electionDate" : ISODate("2023-03-22T08:34:14Z"),
"configVersion" : 276040,
"self" : true,
"lastHeartbeatMessage" : ""
}
],
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1679474063, 6),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
},
"operationTime" : Timestamp(1679474063, 6)
}
rs0:PRIMARY> rs.config()
{
"_id" : "rs0",
"version" : 276040,
"protocolVersion" : NumberLong(1),
"writeConcernMajorityJournalDefault" : true,
"members" : [
{
"_id" : 0,
"host" : "10.1.95.4:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"catchUpTimeoutMillis" : -1,
"catchUpTakeoverDelayMillis" : 30000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
},
"replicaSetId" : ObjectId("6311ccaab8d114d352e0655e")
}
}
Mongo still reachable under same host:
$ date
Wed Mar 22 08:35:02 UTC 2023
$ curl solidatus-db-0.db:27017
It looks like you are trying to access MongoDB over HTTP on the native driver port.
MongoDB Go driver reports
Error starting change stream. Will retry: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: 10.1.95.24:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp 10.1.95.24:27017: i/o timeout }, ] }
From what I understand, the driver has itself cached the topology with single member at IP address 10.1.95.24
which has now become stale.
Shouldn’t the driver go back and use it’s provided connection string to re-discover the topology & new IP of member i.e. 10.1.95.4
?
Thank you for your time,
Max