Mongo version: 4.4.0
OS: windows
Steps to replicate.
- Setup a PSA cluster
- Brought down primary data node.
- Secondary becomes primary and do data insertion.
- Reconfigure PSA cluster to single node setup.
- Readd arbiter and primary node(previous) from step 2 (Primaryis still down).
- Bring down secondary data node as well. Now there is no data node available.
- Bring up primary node in step 1 and do data insertion.
- Now when trying to bring up seondary node in step 6, it would not start.
Here is the mongodb logs
{"t":{"$date":"2021-06-14T05:14:57.585+00:00"},"s":"I", "c":"CONTROL", "id":23285, "ctx":"main","msg":"Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'"}
{"t":{"$date":"2021-06-14T05:14:58.110+00:00"},"s":"W", "c":"ASIO", "id":22601, "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
{"t":{"$date":"2021-06-14T05:14:58.110+00:00"},"s":"I", "c":"NETWORK", "id":4648602, "ctx":"main","msg":"Implicit TCP FastOpen in use."}
{"t":{"$date":"2021-06-14T05:14:58.111+00:00"},"s":"W", "c":"ASIO", "id":22601, "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
{"t":{"$date":"2021-06-14T05:14:58.111+00:00"},"s":"I", "c":"STORAGE", "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":9596,"port":27018,"dbPath":"Z:/MongoDB/psa/secondary/data","architecture":"64-bit","host":"mongo-ww6-clone"}}
{"t":{"$date":"2021-06-14T05:14:58.112+00:00"},"s":"I", "c":"CONTROL", "id":23398, "ctx":"initandlisten","msg":"Target operating system minimum version","attr":{"targetMinOS":"Windows 7/Windows Server 2008 R2"}}
{"t":{"$date":"2021-06-14T05:14:58.112+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"4.4.0","gitVersion":"563487e100c4215e2dce98d0af2a6a5a2d67c5cf","modules":[],"allocator":"tcmalloc","environment":{"distmod":"windows","distarch":"x86_64","target_arch":"x86_64"}}}}
{"t":{"$date":"2021-06-14T05:14:58.112+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"Microsoft Windows Server 2016","version":"10.0 (build 14393)"}}}
{"t":{"$date":"2021-06-14T05:14:58.112+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"Z:\\MongoDB\\psa\\secondary\\mongod.cfg","net":{"bindIp":"0.0.0.0","port":27018},"processManagement":{"windowsService":{"description":"MongoDb node1","displayName":"MongoDb node1","serviceName":"MongoDb node1"}},"replication":{"oplogSizeMB":20840,"replSetName":"rs0"},"storage":{"dbPath":"Z:\\MongoDB\\psa\\secondary\\data","directoryPerDB":true,"journal":{"enabled":true},"oplogMinRetentionHours":24.0,"wiredTiger":{"engineConfig":{"directoryForIndexes":true}}},"systemLog":{"destination":"file","logAppend":true,"logRotate":"reopen","path":"Z:\\MongoDB\\psa\\secondary\\log\\mongod.log"}}}}
{"t":{"$date":"2021-06-14T05:14:58.118+00:00"},"s":"W", "c":"STORAGE", "id":22271, "ctx":"initandlisten","msg":"Detected unclean shutdown - Lock file is not empty","attr":{"lockFile":"Z:\\MongoDB\\psa\\secondary\\data\\mongod.lock"}}
{"t":{"$date":"2021-06-14T05:14:58.124+00:00"},"s":"I", "c":"STORAGE", "id":22270, "ctx":"initandlisten","msg":"Storage engine to use detected by data files","attr":{"dbpath":"Z:/MongoDB/psa/secondary/data","storageEngine":"wiredTiger"}}
{"t":{"$date":"2021-06-14T05:14:58.124+00:00"},"s":"W", "c":"STORAGE", "id":22302, "ctx":"initandlisten","msg":"Recovering data from the last clean checkpoint."}
{"t":{"$date":"2021-06-14T05:14:58.124+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=15871M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],"}}
{"t":{"$date":"2021-06-14T05:14:58.189+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:188966][9596:140705881807312], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 131 through 132"}}
{"t":{"$date":"2021-06-14T05:14:58.278+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:277962][9596:140705881807312], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 132 through 132"}}
{"t":{"$date":"2021-06-14T05:14:58.381+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:380963][9596:140705881807312], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 131/256 to 132/256"}}
{"t":{"$date":"2021-06-14T05:14:58.381+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:380963][9596:140705881807312], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 131 through 132"}}
{"t":{"$date":"2021-06-14T05:14:58.499+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:498964][9596:140705881807312], file:local/index/1--6186380548073876140.wt, txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 132 through 132"}}
{"t":{"$date":"2021-06-14T05:14:58.587+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:586966][9596:140705881807312], file:local/index/1--6186380548073876140.wt, txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (1622300925, 1)"}}
{"t":{"$date":"2021-06-14T05:14:59.151+00:00"},"s":"I", "c":"STORAGE", "id":4795906, "ctx":"initandlisten","msg":"WiredTiger opened","attr":{"durationMillis":1027}}
{"t":{"$date":"2021-06-14T05:14:59.151+00:00"},"s":"I", "c":"RECOVERY", "id":23987, "ctx":"initandlisten","msg":"WiredTiger recoveryTimestamp","attr":{"recoveryTimestamp":{"$timestamp":{"t":1622300925,"i":1}}}}
{"t":{"$date":"2021-06-14T05:14:59.154+00:00"},"s":"I", "c":"STORAGE", "id":22383, "ctx":"initandlisten","msg":"The size storer reports that the oplog contains","attr":{"numRecords":50756766,"dataSize":21655760203}}
{"t":{"$date":"2021-06-14T05:14:59.154+00:00"},"s":"I", "c":"STORAGE", "id":22386, "ctx":"initandlisten","msg":"Sampling the oplog to determine where to place markers for truncation"}
{"t":{"$date":"2021-06-14T05:14:59.155+00:00"},"s":"I", "c":"STORAGE", "id":22389, "ctx":"initandlisten","msg":"Sampling from the oplog to determine where to place markers for truncation","attr":{"from":{"$timestamp":{"t":1621862402,"i":10097}},"to":{"$timestamp":{"t":1622300947,"i":1}}}}
{"t":{"$date":"2021-06-14T05:14:59.155+00:00"},"s":"I", "c":"STORAGE", "id":22390, "ctx":"initandlisten","msg":"Taking samples and assuming each oplog section contains","attr":{"numSamples":991,"containsNumRecords":512175,"containsNumBytes":218523358}}
{"t":{"$date":"2021-06-14T05:15:00.352+00:00"},"s":"I", "c":"STORAGE", "id":22393, "ctx":"initandlisten","msg":"Oplog sampling complete"}
{"t":{"$date":"2021-06-14T05:15:00.352+00:00"},"s":"I", "c":"STORAGE", "id":22382, "ctx":"initandlisten","msg":"WiredTiger record store oplog processing finished","attr":{"durationMillis":1197}}
{"t":{"$date":"2021-06-14T05:15:00.356+00:00"},"s":"I", "c":"STORAGE", "id":22262, "ctx":"initandlisten","msg":"Timestamp monitor starting"}
{"t":{"$date":"2021-06-14T05:15:00.363+00:00"},"s":"W", "c":"CONTROL", "id":22120, "ctx":"initandlisten","msg":"Access control is not enabled for the database. Read and write access to data and configuration is unrestricted","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.373+00:00"},"s":"I", "c":"STORAGE", "id":20536, "ctx":"initandlisten","msg":"Flow Control is enabled on this deployment"}
{"t":{"$date":"2021-06-14T05:15:00.374+00:00"},"s":"I", "c":"SHARDING", "id":20997, "ctx":"initandlisten","msg":"Refreshed RWC defaults","attr":{"newDefaults":{}}}
{"t":{"$date":"2021-06-14T05:15:00.859+00:00"},"s":"I", "c":"FTDC", "id":20625, "ctx":"initandlisten","msg":"Initializing full-time diagnostic data capture","attr":{"dataDirectory":"Z:/MongoDB/psa/secondary/data/diagnostic.data"}}
{"t":{"$date":"2021-06-14T05:15:00.860+00:00"},"s":"I", "c":"REPL", "id":21529, "ctx":"initandlisten","msg":"Initializing rollback ID","attr":{"rbid":1}}
{"t":{"$date":"2021-06-14T05:15:00.860+00:00"},"s":"I", "c":"REPL", "id":21544, "ctx":"initandlisten","msg":"Recovering from stable timestamp","attr":{"stableTimestamp":{"$timestamp":{"t":1622300925,"i":1}},"topOfOplog":{"ts":{"$timestamp":{"t":1622300947,"i":1}},"t":37},"appliedThrough":{"ts":{"$timestamp":{"t":0,"i":0}},"t":-1},"oplogTruncateAfterPoint":{"$timestamp":{"t":0,"i":0}}}}
{"t":{"$date":"2021-06-14T05:15:00.860+00:00"},"s":"I", "c":"REPL", "id":21545, "ctx":"initandlisten","msg":"Starting recovery oplog application at the stable timestamp","attr":{"stableTimestamp":{"$timestamp":{"t":1622300925,"i":1}}}}
{"t":{"$date":"2021-06-14T05:15:00.860+00:00"},"s":"I", "c":"REPL", "id":21550, "ctx":"initandlisten","msg":"Replaying stored operations from startPoint (inclusive) to endPoint (inclusive)","attr":{"startPoint":{"$timestamp":{"t":1622300925,"i":1}},"endPoint":{"$timestamp":{"t":1622300947,"i":1}}}}
{"t":{"$date":"2021-06-14T05:15:00.862+00:00"},"s":"I", "c":"REPL", "id":21536, "ctx":"initandlisten","msg":"Completed oplog application for recovery","attr":{"numOpsApplied":3,"numBatches":1,"applyThroughOpTime":{"ts":{"$timestamp":{"t":1622300947,"i":1}},"t":37}}}
{"t":{"$date":"2021-06-14T05:15:00.864+00:00"},"s":"I", "c":"CONTROL", "id":20714, "ctx":"LogicalSessionCacheRefresh","msg":"Failed to refresh session cache, will try again at the next refresh interval","attr":{"error":"NotYetInitialized: Replication has not yet been configured"}}
{"t":{"$date":"2021-06-14T05:15:00.864+00:00"},"s":"I", "c":"REPL", "id":40440, "ctx":"initandlisten","msg":"Starting the TopologyVersionObserver"}
{"t":{"$date":"2021-06-14T05:15:00.864+00:00"},"s":"I", "c":"CONTROL", "id":20711, "ctx":"LogicalSessionCacheReap","msg":"Failed to reap transaction table","attr":{"error":"NotYetInitialized: Replication has not yet been configured"}}
{"t":{"$date":"2021-06-14T05:15:00.865+00:00"},"s":"I", "c":"REPL", "id":40445, "ctx":"TopologyVersionObserver","msg":"Started TopologyVersionObserver"}
{"t":{"$date":"2021-06-14T05:15:00.865+00:00"},"s":"I", "c":"NETWORK", "id":23015, "ctx":"listener","msg":"Listening on","attr":{"address":"0.0.0.0"}}
{"t":{"$date":"2021-06-14T05:15:00.865+00:00"},"s":"I", "c":"NETWORK", "id":23016, "ctx":"listener","msg":"Waiting for connections","attr":{"port":27018,"ssl":"off"}}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I", "c":"REPL", "id":21315, "ctx":"ReplCoord-0","msg":"","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I", "c":"REPL", "id":21316, "ctx":"ReplCoord-0","msg":"** WARNING: This replica set has a Primary-Secondary-Arbiter architecture, but readConcern:majority is enabled ","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I", "c":"REPL", "id":21317, "ctx":"ReplCoord-0","msg":"** for this node. This is not a recommended configuration. Please see ","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I", "c":"REPL", "id":21318, "ctx":"ReplCoord-0","msg":"** https://dochub.mongodb.org/core/psa-disable-rc-majority","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I", "c":"REPL", "id":21319, "ctx":"ReplCoord-0","msg":"","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I", "c":"REPL", "id":21392, "ctx":"ReplCoord-0","msg":"New replica set config in use","attr":{"config":{"_id":"rs0","version":483019,"term":38,"protocolVersion":1,"writeConcernMajorityJournalDefault":true,"members":[{"_id":9,"host":"mongo-ww6-clone:27018","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0.5,"tags":{},"slaveDelay":0,"votes":1},{"_id":10,"host":"mongo-ww6-clone:27019","arbiterOnly":true,"buildIndexes":true,"hidden":false,"priority":0.0,"tags":{},"slaveDelay":0,"votes":1},{"_id":11,"host":"mongo-ww6-clone:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":1.0,"tags":{},"slaveDelay":0,"votes":1}],"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"catchUpTakeoverDelayMillis":30000,"getLastErrorModes":{},"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":{"$oid":"609e27f78218b48d674cc6f1"}}}}}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I", "c":"REPL", "id":21393, "ctx":"ReplCoord-0","msg":"Found self in config","attr":{"hostAndPort":"mongo-ww6-clone:27018"}}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I", "c":"REPL", "id":21358, "ctx":"ReplCoord-0","msg":"Replica set state transition","attr":{"newState":"STARTUP2","oldState":"STARTUP"}}
{"t":{"$date":"2021-06-14T05:15:00.878+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"connection accepted","attr":{"remote":"10.0.10.5:58240","sessionId":3,"connectionCount":1}}
{"t":{"$date":"2021-06-14T05:15:00.878+00:00"},"s":"I", "c":"REPL", "id":21306, "ctx":"ReplCoord-0","msg":"Starting replication storage threads"}
{"t":{"$date":"2021-06-14T05:15:00.878+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"ReplNetwork","msg":"Connecting","attr":{"hostAndPort":"mongo-ww6-clone:27019"}}
{"t":{"$date":"2021-06-14T05:15:00.878+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn3","msg":"client metadata","attr":{"remote":"10.0.10.5:58240","client":"conn3","doc":{"driver":{"name":"NetworkInterfaceTL","version":"4.4.0"},"os":{"type":"Windows","name":"Microsoft Windows Server 2016","architecture":"x86_64","version":"10.0 (build 14393)"}}}}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"ReplNetwork","msg":"Connecting","attr":{"hostAndPort":"mongo-ww6-clone:27017"}}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I", "c":"REPL", "id":21358, "ctx":"ReplCoord-0","msg":"Replica set state transition","attr":{"newState":"RECOVERING","oldState":"STARTUP2"}}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I", "c":"REPL", "id":21299, "ctx":"ReplCoord-0","msg":"Starting replication fetcher thread"}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I", "c":"REPL", "id":21300, "ctx":"ReplCoord-0","msg":"Starting replication applier thread"}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I", "c":"REPL", "id":21301, "ctx":"ReplCoord-0","msg":"Starting replication reporter thread"}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I", "c":"REPL", "id":21224, "ctx":"OplogApplier-0","msg":"Starting oplog application"}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I", "c":"REPL", "id":21358, "ctx":"OplogApplier-0","msg":"Replica set state transition","attr":{"newState":"SECONDARY","oldState":"RECOVERING"}}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I", "c":"REPL", "id":21106, "ctx":"OplogApplier-0","msg":"Resetting sync source to empty","attr":{"previousSyncSource":":27017"}}
{"t":{"$date":"2021-06-14T05:15:00.880+00:00"},"s":"I", "c":"REPL", "id":21215, "ctx":"ReplCoord-1","msg":"Member is in new state","attr":{"hostAndPort":"mongo-ww6-clone:27019","newState":"ARBITER"}}
{"t":{"$date":"2021-06-14T05:15:00.880+00:00"},"s":"I", "c":"REPL", "id":21215, "ctx":"ReplCoord-0","msg":"Member is in new state","attr":{"hostAndPort":"mongo-ww6-clone:27017","newState":"PRIMARY"}}
{"t":{"$date":"2021-06-14T05:15:00.881+00:00"},"s":"I", "c":"REPL", "id":21783, "ctx":"BackgroundSync","msg":"Waiting for pings from other members before syncing","attr":{"pingsNeeded":2}}
{"t":{"$date":"2021-06-14T05:15:01.004+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"connection accepted","attr":{"remote":"10.0.10.4:64008","sessionId":6,"connectionCount":2}}
{"t":{"$date":"2021-06-14T05:15:01.005+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn6","msg":"client metadata","attr":{"remote":"10.0.10.4:64008","client":"conn6","doc":{"driver":{"name":"mongo-go-driver","version":"v1.3.5"},"os":{"type":"windows","architecture":"amd64"},"platform":"go1.14.1","application":{"name":"monstache"}}}}
{"t":{"$date":"2021-06-14T05:15:01.150+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"connection accepted","attr":{"remote":"10.0.10.5:58242","sessionId":7,"connectionCount":3}}
{"t":{"$date":"2021-06-14T05:15:01.150+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn7","msg":"client metadata","attr":{"remote":"10.0.10.5:58242","client":"conn7","doc":{"driver":{"name":"NetworkInterfaceTL","version":"4.4.0"},"os":{"type":"Windows","name":"Microsoft Windows Server 2016","architecture":"x86_64","version":"10.0 (build 14393)"}}}}
{"t":{"$date":"2021-06-14T05:15:01.364+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"connection accepted","attr":{"remote":"10.0.10.5:58243","sessionId":8,"connectionCount":4}}
{"t":{"$date":"2021-06-14T05:15:01.365+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn8","msg":"client metadata","attr":{"remote":"10.0.10.5:58243","client":"conn8","doc":{"driver":{"name":"NetworkInterfaceTL","version":"4.4.0"},"os":{"type":"Windows","name":"Microsoft Windows Server 2016","architecture":"x86_64","version":"10.0 (build 14393)"}}}}
{"t":{"$date":"2021-06-14T05:15:01.882+00:00"},"s":"I", "c":"REPL", "id":21799, "ctx":"BackgroundSync","msg":"Sync source candidate chosen","attr":{"syncSource":"mongo-ww6-clone:27017"}}
{"t":{"$date":"2021-06-14T05:15:01.882+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"ReplCoordExternNetwork","msg":"Connecting","attr":{"hostAndPort":"mongo-ww6-clone:27017"}}
{"t":{"$date":"2021-06-14T05:15:01.884+00:00"},"s":"I", "c":"REPL", "id":21088, "ctx":"BackgroundSync","msg":"Changed sync source","attr":{"oldSyncSource":"empty","newSyncSource":"mongo-ww6-clone:27017"}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I", "c":"REPL", "id":21098, "ctx":"BackgroundSync","msg":"Starting rollback due to fetcher error","attr":{"error":"OplogStartMissing: Our last optime fetched: { ts: Timestamp(1622300947, 1), t: 37 }. source's GTE: { ts: Timestamp(1622301515, 1), t: 38 }","lastCommittedOpTime":{"ts":{"$timestamp":{"t":0,"i":0}},"t":-1}}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I", "c":"REPL", "id":21102, "ctx":"BackgroundSync","msg":"Rollback using 'recoverToStableTimestamp' method"}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I", "c":"REPL", "id":21104, "ctx":"BackgroundSync","msg":"Scheduling rollback","attr":{"syncSource":"mongo-ww6-clone:27017"}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I", "c":"ROLLBACK", "id":21593, "ctx":"BackgroundSync","msg":"Transition to ROLLBACK"}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I", "c":"REPL", "id":21340, "ctx":"BackgroundSync","msg":"State transition ops metrics","attr":{"metrics":{"lastStateTransition":"rollback","userOpsKilled":0,"userOpsRunning":5}}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I", "c":"REPL", "id":21358, "ctx":"BackgroundSync","msg":"Replica set state transition","attr":{"newState":"ROLLBACK","oldState":"SECONDARY"}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I", "c":"NETWORK", "id":22991, "ctx":"BackgroundSync","msg":"Skip closing connection for connection","attr":{"connectionId":8}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I", "c":"NETWORK", "id":22991, "ctx":"BackgroundSync","msg":"Skip closing connection for connection","attr":{"connectionId":7}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I", "c":"NETWORK", "id":22991, "ctx":"BackgroundSync","msg":"Skip closing connection for connection","attr":{"connectionId":3}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I", "c":"ROLLBACK", "id":21606, "ctx":"BackgroundSync","msg":"Finding common point"}
{"t":{"$date":"2021-06-14T05:15:02.029+00:00"},"s":"I", "c":"NETWORK", "id":22990, "ctx":"conn7","msg":"DBException handling request, closing client connection","attr":{"error":"NotMaster: Not-master error while processing 'find' operation on 'local' database via fire-and-forget command execution."}}
{"t":{"$date":"2021-06-14T05:15:02.029+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn7","msg":"connection ended","attr":{"remote":"10.0.10.5:58242","connectionCount":3}}
{"t":{"$date":"2021-06-14T05:15:02.385+00:00"},"s":"I", "c":"ROLLBACK", "id":21607, "ctx":"BackgroundSync","msg":"Rollback common point","attr":{"commonPointOpTime":{"ts":{"$timestamp":{"t":1622300653,"i":1}},"t":35}}}
{"t":{"$date":"2021-06-14T05:15:02.385+00:00"},"s":"F", "c":"ROLLBACK", "id":51121, "ctx":"BackgroundSync","msg":"Common point must be at least stable timestamp","attr":{"commonPoint":{"$timestamp":{"t":1622300653,"i":1}},"stableTimestamp":{"$timestamp":{"t":1622300925,"i":1}}}}
{"t":{"$date":"2021-06-14T05:15:02.386+00:00"},"s":"F", "c":"-", "id":23091, "ctx":"BackgroundSync","msg":"Fatal assertion","attr":{"msgid":51121,"file":"src\\mongo\\db\\repl\\rollback_impl.cpp","line":1035}}
{"t":{"$date":"2021-06-14T05:15:02.386+00:00"},"s":"F", "c":"-", "id":23092, "ctx":"BackgroundSync","msg":"\n\n***aborting after fassert() failure\n\n"}
>
> rs0:PRIMARY> rs.status()
> {
> "set" : "rs0",
> "date" : ISODate("2021-06-14T05:16:08.838Z"),
> "myState" : 1,
> "term" : NumberLong(38),
> "syncSourceHost" : "",
> "syncSourceId" : -1,
> "heartbeatIntervalMillis" : NumberLong(2000),
> "majorityVoteCount" : 2,
> "writeMajorityCount" : 2,
> "votingMembersCount" : 3,
> "writableVotingMembersCount" : 2,
> "optimes" : {
> "lastCommittedOpTime" : {
> "ts" : Timestamp(0, 0),
> "t" : NumberLong(-1)
> },
> "lastCommittedWallTime" : ISODate("1970-01-01T00:00:00Z"),
> "appliedOpTime" : {
> "ts" : Timestamp(1623647760, 1),
> "t" : NumberLong(38)
> },
> "durableOpTime" : {
> "ts" : Timestamp(1623647760, 1),
> "t" : NumberLong(38)
> },
> "lastAppliedWallTime" : ISODate("2021-06-14T05:16:00.082Z"),
> "lastDurableWallTime" : ISODate("2021-06-14T05:16:00.082Z")
> },
> "lastStableRecoveryTimestamp" : Timestamp(1622300653, 1),
> "electionCandidateMetrics" : {
> "lastElectionReason" : "electionTimeout",
> "lastElectionDate" : ISODate("2021-05-29T15:18:34.315Z"),
> "electionTerm" : NumberLong(38),
> "lastCommittedOpTimeAtElection" : {
> "ts" : Timestamp(0, 0),
> "t" : NumberLong(-1)
> },
> "lastSeenOpTimeAtElection" : {
> "ts" : Timestamp(1622300653, 1),
> "t" : NumberLong(35)
> },
> "numVotesNeeded" : 2,
> "priorityAtElection" : 1,
> "electionTimeoutMillis" : NumberLong(10000),
> "numCatchUpOps" : NumberLong(0),
> "newTermStartDate" : ISODate("2021-05-29T15:18:35.322Z")
> },
> "members" : [
> {
> "_id" : 9,
> "name" : "mongo-ww6-clone:27018",
> "health" : 0,
> "state" : 8,
> "stateStr" : "(not reachable/healthy)",
> "uptime" : 0,
> "optime" : {
> "ts" : Timestamp(0, 0),
> "t" : NumberLong(-1)
> },
> "optimeDurable" : {
> "ts" : Timestamp(0, 0),
> "t" : NumberLong(-1)
> },
> "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
> "optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
> "lastHeartbeat" : ISODate("2021-06-14T05:16:05.955Z"),
> "lastHeartbeatRecv" : ISODate("2021-06-14T05:15:01.882Z"),
> "pingMs" : NumberLong(0),
> "lastHeartbeatMessage" : "Error connecting to mongo-ww6-clone:27018 (10.0.10.5:27018) :: caused by :: No connection could be made because the target machine actively refused it.",
> "syncSourceHost" : "",
> "syncSourceId" : -1,
> "infoMessage" : "",
> "configVersion" : 483019,
> "configTerm" : 38
> },
> {
> "_id" : 10,
> "name" : "mongo-ww6-clone:27019",
> "health" : 1,
> "state" : 7,
> "stateStr" : "ARBITER",
> "uptime" : 1346256,
> "lastHeartbeat" : ISODate("2021-06-14T05:16:07.513Z"),
> "lastHeartbeatRecv" : ISODate("2021-06-14T05:16:07.513Z"),
> "pingMs" : NumberLong(0),
> "lastHeartbeatMessage" : "",
> "syncSourceHost" : "",
> "syncSourceId" : -1,
> "infoMessage" : "",
> "configVersion" : 483019,
> "configTerm" : 38
> },
> {
> "_id" : 11,
> "name" : "mongo-ww6-clone:27017",
> "health" : 1,
> "state" : 1,
> "stateStr" : "PRIMARY",
> "uptime" : 1346320,
> "optime" : {
> "ts" : Timestamp(1623647760, 1),
> "t" : NumberLong(38)
> },
> "optimeDate" : ISODate("2021-06-14T05:16:00Z"),
> "syncSourceHost" : "",
> "syncSourceId" : -1,
> "infoMessage" : "",
> "electionTime" : Timestamp(1622301514, 1),
> "electionDate" : ISODate("2021-05-29T15:18:34Z"),
> "configVersion" : 483019,
> "configTerm" : 38,
> "self" : true,
> "lastHeartbeatMessage" : ""
> }
> ],
> "ok" : 1,
> "$clusterTime" : {
> "clusterTime" : Timestamp(1623647760, 1),
> "signature" : {
> "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
> "keyId" : NumberLong(0)
> }
> },
> "operationTime" : Timestamp(1623647760, 1)
> }
> rs0:PRIMARY>