Hi friends,
I have mongodb replicaset installed on my original host (k8s cluster 1) and i have just installed a new mongodb replicaset on the new host (k8s cluster 2).
My goal is to replicate the data from the original host to the new host, so that i can shutdown the old host.
There is a blog post that im trying to follow to achieve this however im having some trouble. (reference: Migrating a MongoBD replica set to a new hosting provider with no downtime | Michael Schmitt | Mobile & Web Developer)
original_host: aadfad22ca65e4ff09de37179f961d5b-.us-east-2.elb.amazonaws.com:27017
new_host: ab775d626d81742478af2744923e2ec6-.us-east-2.elb.amazonaws.com:27017
ATTEMPT
- I tried to step down the new host with
rs.stepDown({force:true}). However it fails…
rs.stepDown({force:true})
{
"ok" : 0,
"errmsg" : "No electable secondaries caught up as of 2022-05-24T08:50:24.809+00:00. Please use the replSetStepDown command with the argument {force: true} to force node to step down.",
"code" : 262,
"codeName" : "ExceededTimeLimit",
"$clusterTime" : {
"clusterTime" : Timestamp(1653382215, 1),
"signature" : {
"hash" : BinData(0,"qugjVF4xVS8+MNYlCgkK+0/Jt1o="),
"keyId" : NumberLong("7100922576402120709")
}
},
"operationTime" : Timestamp(1653382211, 1)
}
- I tried to add the new_host as a rs member to the original rs using
rs.add( { host: "ab775d626d81742478af2744923e2ec6-<redacted>.us-east-2.elb.amazonaws.com:27017", priority: 0, votes: 0 } )However it fails withreplica set IDs do not match
{
"_id" : 2,
"name" : "ab775d626d81742478af2744923e2ec6-<redacted>.us-east-2.elb.amazonaws.com:27017",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDurable" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
"lastAppliedWallTime" : ISODate("1970-01-01T00:00:00Z"),
"lastDurableWallTime" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2022-05-24T08:52:21.889Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "replica set IDs do not match, ours: 628c90e7c07f5017faff8b75; remote node's: 628b8b76ab0cdc7f9158b23b",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : -1,
"configTerm" : -1
}
original replicaset config
rs0:PRIMARY> rs.status()
{
"set" : "rs0",
"date" : ISODate("2022-05-24T08:45:48.912Z"),
"myState" : 1,
"term" : NumberLong(2),
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"majorityVoteCount" : 2,
"writeMajorityCount" : 1,
"votingMembersCount" : 2,
"writableVotingMembersCount" : 1,
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1653381945, 1),
"t" : NumberLong(2)
},
"lastCommittedWallTime" : ISODate("2022-05-24T08:45:45.660Z"),
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1653381945, 1),
"t" : NumberLong(2)
},
"appliedOpTime" : {
"ts" : Timestamp(1653381945, 1),
"t" : NumberLong(2)
},
"durableOpTime" : {
"ts" : Timestamp(1653381945, 1),
"t" : NumberLong(2)
},
"lastAppliedWallTime" : ISODate("2022-05-24T08:45:45.660Z"),
"lastDurableWallTime" : ISODate("2022-05-24T08:45:45.660Z")
},
"lastStableRecoveryTimestamp" : Timestamp(1653381945, 1),
"electionCandidateMetrics" : {
"lastElectionReason" : "electionTimeout",
"lastElectionDate" : ISODate("2022-05-24T08:01:45.606Z"),
"electionTerm" : NumberLong(2),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1653379303, 15),
"t" : NumberLong(1)
},
"numVotesNeeded" : 1,
"priorityAtElection" : 5,
"electionTimeoutMillis" : NumberLong(10000),
"newTermStartDate" : ISODate("2022-05-24T08:01:45.609Z"),
"wMajorityWriteAvailabilityDate" : ISODate("2022-05-24T08:01:45.611Z")
},
"members" : [
{
"_id" : 0,
"name" : "aadfad22ca65e4ff09de37179f961d5b-<redacted>.us-east-2.elb.amazonaws.com:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 2644,
"optime" : {
"ts" : Timestamp(1653381945, 1),
"t" : NumberLong(2)
},
"optimeDate" : ISODate("2022-05-24T08:45:45Z"),
"lastAppliedWallTime" : ISODate("2022-05-24T08:45:45.660Z"),
"lastDurableWallTime" : ISODate("2022-05-24T08:45:45.660Z"),
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1653379305, 1),
"electionDate" : ISODate("2022-05-24T08:01:45Z"),
"configVersion" : 7,
"configTerm" : 2,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 1,
"name" : "mongodb-staging-arbiter-0.mongodb-staging-arbiter-headless.staging.svc.cluster.local:27017",
"health" : 1,
"state" : 7,
"stateStr" : "ARBITER",
"uptime" : 2627,
"lastHeartbeat" : ISODate("2022-05-24T08:45:47.161Z"),
"lastHeartbeatRecv" : ISODate("2022-05-24T08:45:47.763Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : 7,
"configTerm" : 2
},
{
"_id" : 2,
"name" : "ab775d626d81742478af2744923e2ec6-<redacted>.us-east-2.elb.amazonaws.com:27017",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDurable" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
"lastAppliedWallTime" : ISODate("1970-01-01T00:00:00Z"),
"lastDurableWallTime" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2022-05-24T08:45:47.280Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "replica set IDs do not match, ours: 628c90e7c07f5017faff8b75; remote node's: 628b8b76ab0cdc7f9158b23b",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : -1,
"configTerm" : -1
}
],
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1653381945, 1),
"signature" : {
"hash" : BinData(0,"1GVq5OEAV3YzQmhPHuzQPVwC7+c="),
"keyId" : NumberLong("7101210034268274693")
}
},
"operationTime" : Timestamp(1653381945, 1)
}
new replicaset config
{
"set" : "rs0",
"date" : ISODate("2022-05-24T08:45:52.621Z"),
"myState" : 1,
"term" : NumberLong(6),
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"majorityVoteCount" : 2,
"writeMajorityCount" : 1,
"votingMembersCount" : 2,
"writableVotingMembersCount" : 1,
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1653381951, 1),
"t" : NumberLong(6)
},
"lastCommittedWallTime" : ISODate("2022-05-24T08:45:51.410Z"),
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1653381951, 1),
"t" : NumberLong(6)
},
"appliedOpTime" : {
"ts" : Timestamp(1653381951, 1),
"t" : NumberLong(6)
},
"durableOpTime" : {
"ts" : Timestamp(1653381951, 1),
"t" : NumberLong(6)
},
"lastAppliedWallTime" : ISODate("2022-05-24T08:45:51.410Z"),
"lastDurableWallTime" : ISODate("2022-05-24T08:45:51.410Z")
},
"lastStableRecoveryTimestamp" : Timestamp(1653381921, 1),
"electionCandidateMetrics" : {
"lastElectionReason" : "electionTimeout",
"lastElectionDate" : ISODate("2022-05-24T08:42:31.402Z"),
"electionTerm" : NumberLong(6),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1653381732, 1),
"t" : NumberLong(5)
},
"numVotesNeeded" : 1,
"priorityAtElection" : 5,
"electionTimeoutMillis" : NumberLong(10000),
"newTermStartDate" : ISODate("2022-05-24T08:42:31.405Z"),
"wMajorityWriteAvailabilityDate" : ISODate("2022-05-24T08:42:31.407Z")
},
"members" : [
{
"_id" : 0,
"name" : "ab775d626d81742478af2744923e2ec6-<redacted>.us-east-2.elb.amazonaws.com:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 204,
"optime" : {
"ts" : Timestamp(1653381951, 1),
"t" : NumberLong(6)
},
"optimeDate" : ISODate("2022-05-24T08:45:51Z"),
"lastAppliedWallTime" : ISODate("2022-05-24T08:45:51.410Z"),
"lastDurableWallTime" : ISODate("2022-05-24T08:45:51.410Z"),
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1653381751, 1),
"electionDate" : ISODate("2022-05-24T08:42:31Z"),
"configVersion" : 4,
"configTerm" : 6,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 1,
"name" : "mongodb-prod-arbiter-0.mongodb-prod-arbiter-headless.mongodb.svc.cluster.local:27017",
"health" : 1,
"state" : 7,
"stateStr" : "ARBITER",
"uptime" : 187,
"lastHeartbeat" : ISODate("2022-05-24T08:45:50.898Z"),
"lastHeartbeatRecv" : ISODate("2022-05-24T08:45:52.493Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : 4,
"configTerm" : 6
}
],
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1653381951, 1),
"signature" : {
"hash" : BinData(0,"XYfuUbFoIwFfLZx3FqPOAD+CU44="),
"keyId" : NumberLong("7100922576402120709")
}
},
"operationTime" : Timestamp(1653381951, 1)
}