How to recover from Mongodb crashing because of rollback failure

Mongo version: 4.4.0
OS: windows

Steps to replicate.

  1. Setup a PSA cluster
  2. Brought down primary data node.
  3. Secondary becomes primary and do data insertion.
  4. Reconfigure PSA cluster to single node setup.
  5. Readd arbiter and primary node(previous) from step 2 (Primaryis still down).
  6. Bring down secondary data node as well. Now there is no data node available.
  7. Bring up primary node in step 1 and do data insertion.
  8. Now when trying to bring up seondary node in step 6, it would not start.

Here is the mongodb logs

{"t":{"$date":"2021-06-14T05:14:57.585+00:00"},"s":"I",  "c":"CONTROL",  "id":23285,   "ctx":"main","msg":"Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'"}
{"t":{"$date":"2021-06-14T05:14:58.110+00:00"},"s":"W",  "c":"ASIO",     "id":22601,   "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
{"t":{"$date":"2021-06-14T05:14:58.110+00:00"},"s":"I",  "c":"NETWORK",  "id":4648602, "ctx":"main","msg":"Implicit TCP FastOpen in use."}
{"t":{"$date":"2021-06-14T05:14:58.111+00:00"},"s":"W",  "c":"ASIO",     "id":22601,   "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
{"t":{"$date":"2021-06-14T05:14:58.111+00:00"},"s":"I",  "c":"STORAGE",  "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":9596,"port":27018,"dbPath":"Z:/MongoDB/psa/secondary/data","architecture":"64-bit","host":"mongo-ww6-clone"}}
{"t":{"$date":"2021-06-14T05:14:58.112+00:00"},"s":"I",  "c":"CONTROL",  "id":23398,   "ctx":"initandlisten","msg":"Target operating system minimum version","attr":{"targetMinOS":"Windows 7/Windows Server 2008 R2"}}
{"t":{"$date":"2021-06-14T05:14:58.112+00:00"},"s":"I",  "c":"CONTROL",  "id":23403,   "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"4.4.0","gitVersion":"563487e100c4215e2dce98d0af2a6a5a2d67c5cf","modules":[],"allocator":"tcmalloc","environment":{"distmod":"windows","distarch":"x86_64","target_arch":"x86_64"}}}}
{"t":{"$date":"2021-06-14T05:14:58.112+00:00"},"s":"I",  "c":"CONTROL",  "id":51765,   "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"Microsoft Windows Server 2016","version":"10.0 (build 14393)"}}}
{"t":{"$date":"2021-06-14T05:14:58.112+00:00"},"s":"I",  "c":"CONTROL",  "id":21951,   "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"Z:\\MongoDB\\psa\\secondary\\mongod.cfg","net":{"bindIp":"0.0.0.0","port":27018},"processManagement":{"windowsService":{"description":"MongoDb node1","displayName":"MongoDb node1","serviceName":"MongoDb node1"}},"replication":{"oplogSizeMB":20840,"replSetName":"rs0"},"storage":{"dbPath":"Z:\\MongoDB\\psa\\secondary\\data","directoryPerDB":true,"journal":{"enabled":true},"oplogMinRetentionHours":24.0,"wiredTiger":{"engineConfig":{"directoryForIndexes":true}}},"systemLog":{"destination":"file","logAppend":true,"logRotate":"reopen","path":"Z:\\MongoDB\\psa\\secondary\\log\\mongod.log"}}}}
{"t":{"$date":"2021-06-14T05:14:58.118+00:00"},"s":"W",  "c":"STORAGE",  "id":22271,   "ctx":"initandlisten","msg":"Detected unclean shutdown - Lock file is not empty","attr":{"lockFile":"Z:\\MongoDB\\psa\\secondary\\data\\mongod.lock"}}
{"t":{"$date":"2021-06-14T05:14:58.124+00:00"},"s":"I",  "c":"STORAGE",  "id":22270,   "ctx":"initandlisten","msg":"Storage engine to use detected by data files","attr":{"dbpath":"Z:/MongoDB/psa/secondary/data","storageEngine":"wiredTiger"}}
{"t":{"$date":"2021-06-14T05:14:58.124+00:00"},"s":"W",  "c":"STORAGE",  "id":22302,   "ctx":"initandlisten","msg":"Recovering data from the last clean checkpoint."}
{"t":{"$date":"2021-06-14T05:14:58.124+00:00"},"s":"I",  "c":"STORAGE",  "id":22315,   "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=15871M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],"}}
{"t":{"$date":"2021-06-14T05:14:58.189+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:188966][9596:140705881807312], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 131 through 132"}}
{"t":{"$date":"2021-06-14T05:14:58.278+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:277962][9596:140705881807312], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 132 through 132"}}
{"t":{"$date":"2021-06-14T05:14:58.381+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:380963][9596:140705881807312], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 131/256 to 132/256"}}
{"t":{"$date":"2021-06-14T05:14:58.381+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:380963][9596:140705881807312], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 131 through 132"}}
{"t":{"$date":"2021-06-14T05:14:58.499+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:498964][9596:140705881807312], file:local/index/1--6186380548073876140.wt, txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 132 through 132"}}
{"t":{"$date":"2021-06-14T05:14:58.587+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1623647698:586966][9596:140705881807312], file:local/index/1--6186380548073876140.wt, txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (1622300925, 1)"}}
{"t":{"$date":"2021-06-14T05:14:59.151+00:00"},"s":"I",  "c":"STORAGE",  "id":4795906, "ctx":"initandlisten","msg":"WiredTiger opened","attr":{"durationMillis":1027}}
{"t":{"$date":"2021-06-14T05:14:59.151+00:00"},"s":"I",  "c":"RECOVERY", "id":23987,   "ctx":"initandlisten","msg":"WiredTiger recoveryTimestamp","attr":{"recoveryTimestamp":{"$timestamp":{"t":1622300925,"i":1}}}}
{"t":{"$date":"2021-06-14T05:14:59.154+00:00"},"s":"I",  "c":"STORAGE",  "id":22383,   "ctx":"initandlisten","msg":"The size storer reports that the oplog contains","attr":{"numRecords":50756766,"dataSize":21655760203}}
{"t":{"$date":"2021-06-14T05:14:59.154+00:00"},"s":"I",  "c":"STORAGE",  "id":22386,   "ctx":"initandlisten","msg":"Sampling the oplog to determine where to place markers for truncation"}
{"t":{"$date":"2021-06-14T05:14:59.155+00:00"},"s":"I",  "c":"STORAGE",  "id":22389,   "ctx":"initandlisten","msg":"Sampling from the oplog to determine where to place markers for truncation","attr":{"from":{"$timestamp":{"t":1621862402,"i":10097}},"to":{"$timestamp":{"t":1622300947,"i":1}}}}
{"t":{"$date":"2021-06-14T05:14:59.155+00:00"},"s":"I",  "c":"STORAGE",  "id":22390,   "ctx":"initandlisten","msg":"Taking samples and assuming each oplog section contains","attr":{"numSamples":991,"containsNumRecords":512175,"containsNumBytes":218523358}}
{"t":{"$date":"2021-06-14T05:15:00.352+00:00"},"s":"I",  "c":"STORAGE",  "id":22393,   "ctx":"initandlisten","msg":"Oplog sampling complete"}
{"t":{"$date":"2021-06-14T05:15:00.352+00:00"},"s":"I",  "c":"STORAGE",  "id":22382,   "ctx":"initandlisten","msg":"WiredTiger record store oplog processing finished","attr":{"durationMillis":1197}}
{"t":{"$date":"2021-06-14T05:15:00.356+00:00"},"s":"I",  "c":"STORAGE",  "id":22262,   "ctx":"initandlisten","msg":"Timestamp monitor starting"}
{"t":{"$date":"2021-06-14T05:15:00.363+00:00"},"s":"W",  "c":"CONTROL",  "id":22120,   "ctx":"initandlisten","msg":"Access control is not enabled for the database. Read and write access to data and configuration is unrestricted","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.373+00:00"},"s":"I",  "c":"STORAGE",  "id":20536,   "ctx":"initandlisten","msg":"Flow Control is enabled on this deployment"}
{"t":{"$date":"2021-06-14T05:15:00.374+00:00"},"s":"I",  "c":"SHARDING", "id":20997,   "ctx":"initandlisten","msg":"Refreshed RWC defaults","attr":{"newDefaults":{}}}
{"t":{"$date":"2021-06-14T05:15:00.859+00:00"},"s":"I",  "c":"FTDC",     "id":20625,   "ctx":"initandlisten","msg":"Initializing full-time diagnostic data capture","attr":{"dataDirectory":"Z:/MongoDB/psa/secondary/data/diagnostic.data"}}
{"t":{"$date":"2021-06-14T05:15:00.860+00:00"},"s":"I",  "c":"REPL",     "id":21529,   "ctx":"initandlisten","msg":"Initializing rollback ID","attr":{"rbid":1}}
{"t":{"$date":"2021-06-14T05:15:00.860+00:00"},"s":"I",  "c":"REPL",     "id":21544,   "ctx":"initandlisten","msg":"Recovering from stable timestamp","attr":{"stableTimestamp":{"$timestamp":{"t":1622300925,"i":1}},"topOfOplog":{"ts":{"$timestamp":{"t":1622300947,"i":1}},"t":37},"appliedThrough":{"ts":{"$timestamp":{"t":0,"i":0}},"t":-1},"oplogTruncateAfterPoint":{"$timestamp":{"t":0,"i":0}}}}
{"t":{"$date":"2021-06-14T05:15:00.860+00:00"},"s":"I",  "c":"REPL",     "id":21545,   "ctx":"initandlisten","msg":"Starting recovery oplog application at the stable timestamp","attr":{"stableTimestamp":{"$timestamp":{"t":1622300925,"i":1}}}}
{"t":{"$date":"2021-06-14T05:15:00.860+00:00"},"s":"I",  "c":"REPL",     "id":21550,   "ctx":"initandlisten","msg":"Replaying stored operations from startPoint (inclusive) to endPoint (inclusive)","attr":{"startPoint":{"$timestamp":{"t":1622300925,"i":1}},"endPoint":{"$timestamp":{"t":1622300947,"i":1}}}}
{"t":{"$date":"2021-06-14T05:15:00.862+00:00"},"s":"I",  "c":"REPL",     "id":21536,   "ctx":"initandlisten","msg":"Completed oplog application for recovery","attr":{"numOpsApplied":3,"numBatches":1,"applyThroughOpTime":{"ts":{"$timestamp":{"t":1622300947,"i":1}},"t":37}}}
{"t":{"$date":"2021-06-14T05:15:00.864+00:00"},"s":"I",  "c":"CONTROL",  "id":20714,   "ctx":"LogicalSessionCacheRefresh","msg":"Failed to refresh session cache, will try again at the next refresh interval","attr":{"error":"NotYetInitialized: Replication has not yet been configured"}}
{"t":{"$date":"2021-06-14T05:15:00.864+00:00"},"s":"I",  "c":"REPL",     "id":40440,   "ctx":"initandlisten","msg":"Starting the TopologyVersionObserver"}
{"t":{"$date":"2021-06-14T05:15:00.864+00:00"},"s":"I",  "c":"CONTROL",  "id":20711,   "ctx":"LogicalSessionCacheReap","msg":"Failed to reap transaction table","attr":{"error":"NotYetInitialized: Replication has not yet been configured"}}
{"t":{"$date":"2021-06-14T05:15:00.865+00:00"},"s":"I",  "c":"REPL",     "id":40445,   "ctx":"TopologyVersionObserver","msg":"Started TopologyVersionObserver"}
{"t":{"$date":"2021-06-14T05:15:00.865+00:00"},"s":"I",  "c":"NETWORK",  "id":23015,   "ctx":"listener","msg":"Listening on","attr":{"address":"0.0.0.0"}}
{"t":{"$date":"2021-06-14T05:15:00.865+00:00"},"s":"I",  "c":"NETWORK",  "id":23016,   "ctx":"listener","msg":"Waiting for connections","attr":{"port":27018,"ssl":"off"}}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I",  "c":"REPL",     "id":21315,   "ctx":"ReplCoord-0","msg":"","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I",  "c":"REPL",     "id":21316,   "ctx":"ReplCoord-0","msg":"** WARNING: This replica set has a Primary-Secondary-Arbiter architecture, but readConcern:majority is enabled ","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I",  "c":"REPL",     "id":21317,   "ctx":"ReplCoord-0","msg":"**          for this node. This is not a recommended configuration. Please see ","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I",  "c":"REPL",     "id":21318,   "ctx":"ReplCoord-0","msg":"**          https://dochub.mongodb.org/core/psa-disable-rc-majority","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I",  "c":"REPL",     "id":21319,   "ctx":"ReplCoord-0","msg":"","tags":["startupWarnings"]}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I",  "c":"REPL",     "id":21392,   "ctx":"ReplCoord-0","msg":"New replica set config in use","attr":{"config":{"_id":"rs0","version":483019,"term":38,"protocolVersion":1,"writeConcernMajorityJournalDefault":true,"members":[{"_id":9,"host":"mongo-ww6-clone:27018","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0.5,"tags":{},"slaveDelay":0,"votes":1},{"_id":10,"host":"mongo-ww6-clone:27019","arbiterOnly":true,"buildIndexes":true,"hidden":false,"priority":0.0,"tags":{},"slaveDelay":0,"votes":1},{"_id":11,"host":"mongo-ww6-clone:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":1.0,"tags":{},"slaveDelay":0,"votes":1}],"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"catchUpTakeoverDelayMillis":30000,"getLastErrorModes":{},"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":{"$oid":"609e27f78218b48d674cc6f1"}}}}}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I",  "c":"REPL",     "id":21393,   "ctx":"ReplCoord-0","msg":"Found self in config","attr":{"hostAndPort":"mongo-ww6-clone:27018"}}
{"t":{"$date":"2021-06-14T05:15:00.877+00:00"},"s":"I",  "c":"REPL",     "id":21358,   "ctx":"ReplCoord-0","msg":"Replica set state transition","attr":{"newState":"STARTUP2","oldState":"STARTUP"}}
{"t":{"$date":"2021-06-14T05:15:00.878+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"connection accepted","attr":{"remote":"10.0.10.5:58240","sessionId":3,"connectionCount":1}}
{"t":{"$date":"2021-06-14T05:15:00.878+00:00"},"s":"I",  "c":"REPL",     "id":21306,   "ctx":"ReplCoord-0","msg":"Starting replication storage threads"}
{"t":{"$date":"2021-06-14T05:15:00.878+00:00"},"s":"I",  "c":"CONNPOOL", "id":22576,   "ctx":"ReplNetwork","msg":"Connecting","attr":{"hostAndPort":"mongo-ww6-clone:27019"}}
{"t":{"$date":"2021-06-14T05:15:00.878+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn3","msg":"client metadata","attr":{"remote":"10.0.10.5:58240","client":"conn3","doc":{"driver":{"name":"NetworkInterfaceTL","version":"4.4.0"},"os":{"type":"Windows","name":"Microsoft Windows Server 2016","architecture":"x86_64","version":"10.0 (build 14393)"}}}}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I",  "c":"CONNPOOL", "id":22576,   "ctx":"ReplNetwork","msg":"Connecting","attr":{"hostAndPort":"mongo-ww6-clone:27017"}}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I",  "c":"REPL",     "id":21358,   "ctx":"ReplCoord-0","msg":"Replica set state transition","attr":{"newState":"RECOVERING","oldState":"STARTUP2"}}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I",  "c":"REPL",     "id":21299,   "ctx":"ReplCoord-0","msg":"Starting replication fetcher thread"}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I",  "c":"REPL",     "id":21300,   "ctx":"ReplCoord-0","msg":"Starting replication applier thread"}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I",  "c":"REPL",     "id":21301,   "ctx":"ReplCoord-0","msg":"Starting replication reporter thread"}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I",  "c":"REPL",     "id":21224,   "ctx":"OplogApplier-0","msg":"Starting oplog application"}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I",  "c":"REPL",     "id":21358,   "ctx":"OplogApplier-0","msg":"Replica set state transition","attr":{"newState":"SECONDARY","oldState":"RECOVERING"}}
{"t":{"$date":"2021-06-14T05:15:00.879+00:00"},"s":"I",  "c":"REPL",     "id":21106,   "ctx":"OplogApplier-0","msg":"Resetting sync source to empty","attr":{"previousSyncSource":":27017"}}
{"t":{"$date":"2021-06-14T05:15:00.880+00:00"},"s":"I",  "c":"REPL",     "id":21215,   "ctx":"ReplCoord-1","msg":"Member is in new state","attr":{"hostAndPort":"mongo-ww6-clone:27019","newState":"ARBITER"}}
{"t":{"$date":"2021-06-14T05:15:00.880+00:00"},"s":"I",  "c":"REPL",     "id":21215,   "ctx":"ReplCoord-0","msg":"Member is in new state","attr":{"hostAndPort":"mongo-ww6-clone:27017","newState":"PRIMARY"}}
{"t":{"$date":"2021-06-14T05:15:00.881+00:00"},"s":"I",  "c":"REPL",     "id":21783,   "ctx":"BackgroundSync","msg":"Waiting for pings from other members before syncing","attr":{"pingsNeeded":2}}
{"t":{"$date":"2021-06-14T05:15:01.004+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"connection accepted","attr":{"remote":"10.0.10.4:64008","sessionId":6,"connectionCount":2}}
{"t":{"$date":"2021-06-14T05:15:01.005+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn6","msg":"client metadata","attr":{"remote":"10.0.10.4:64008","client":"conn6","doc":{"driver":{"name":"mongo-go-driver","version":"v1.3.5"},"os":{"type":"windows","architecture":"amd64"},"platform":"go1.14.1","application":{"name":"monstache"}}}}
{"t":{"$date":"2021-06-14T05:15:01.150+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"connection accepted","attr":{"remote":"10.0.10.5:58242","sessionId":7,"connectionCount":3}}
{"t":{"$date":"2021-06-14T05:15:01.150+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn7","msg":"client metadata","attr":{"remote":"10.0.10.5:58242","client":"conn7","doc":{"driver":{"name":"NetworkInterfaceTL","version":"4.4.0"},"os":{"type":"Windows","name":"Microsoft Windows Server 2016","architecture":"x86_64","version":"10.0 (build 14393)"}}}}
{"t":{"$date":"2021-06-14T05:15:01.364+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"connection accepted","attr":{"remote":"10.0.10.5:58243","sessionId":8,"connectionCount":4}}
{"t":{"$date":"2021-06-14T05:15:01.365+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn8","msg":"client metadata","attr":{"remote":"10.0.10.5:58243","client":"conn8","doc":{"driver":{"name":"NetworkInterfaceTL","version":"4.4.0"},"os":{"type":"Windows","name":"Microsoft Windows Server 2016","architecture":"x86_64","version":"10.0 (build 14393)"}}}}
{"t":{"$date":"2021-06-14T05:15:01.882+00:00"},"s":"I",  "c":"REPL",     "id":21799,   "ctx":"BackgroundSync","msg":"Sync source candidate chosen","attr":{"syncSource":"mongo-ww6-clone:27017"}}
{"t":{"$date":"2021-06-14T05:15:01.882+00:00"},"s":"I",  "c":"CONNPOOL", "id":22576,   "ctx":"ReplCoordExternNetwork","msg":"Connecting","attr":{"hostAndPort":"mongo-ww6-clone:27017"}}
{"t":{"$date":"2021-06-14T05:15:01.884+00:00"},"s":"I",  "c":"REPL",     "id":21088,   "ctx":"BackgroundSync","msg":"Changed sync source","attr":{"oldSyncSource":"empty","newSyncSource":"mongo-ww6-clone:27017"}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I",  "c":"REPL",     "id":21098,   "ctx":"BackgroundSync","msg":"Starting rollback due to fetcher error","attr":{"error":"OplogStartMissing: Our last optime fetched: { ts: Timestamp(1622300947, 1), t: 37 }. source's GTE: { ts: Timestamp(1622301515, 1), t: 38 }","lastCommittedOpTime":{"ts":{"$timestamp":{"t":0,"i":0}},"t":-1}}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I",  "c":"REPL",     "id":21102,   "ctx":"BackgroundSync","msg":"Rollback using 'recoverToStableTimestamp' method"}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I",  "c":"REPL",     "id":21104,   "ctx":"BackgroundSync","msg":"Scheduling rollback","attr":{"syncSource":"mongo-ww6-clone:27017"}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I",  "c":"ROLLBACK", "id":21593,   "ctx":"BackgroundSync","msg":"Transition to ROLLBACK"}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I",  "c":"REPL",     "id":21340,   "ctx":"BackgroundSync","msg":"State transition ops metrics","attr":{"metrics":{"lastStateTransition":"rollback","userOpsKilled":0,"userOpsRunning":5}}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I",  "c":"REPL",     "id":21358,   "ctx":"BackgroundSync","msg":"Replica set state transition","attr":{"newState":"ROLLBACK","oldState":"SECONDARY"}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I",  "c":"NETWORK",  "id":22991,   "ctx":"BackgroundSync","msg":"Skip closing connection for connection","attr":{"connectionId":8}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I",  "c":"NETWORK",  "id":22991,   "ctx":"BackgroundSync","msg":"Skip closing connection for connection","attr":{"connectionId":7}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I",  "c":"NETWORK",  "id":22991,   "ctx":"BackgroundSync","msg":"Skip closing connection for connection","attr":{"connectionId":3}}
{"t":{"$date":"2021-06-14T05:15:02.014+00:00"},"s":"I",  "c":"ROLLBACK", "id":21606,   "ctx":"BackgroundSync","msg":"Finding common point"}
{"t":{"$date":"2021-06-14T05:15:02.029+00:00"},"s":"I",  "c":"NETWORK",  "id":22990,   "ctx":"conn7","msg":"DBException handling request, closing client connection","attr":{"error":"NotMaster: Not-master error while processing 'find' operation  on 'local' database via fire-and-forget command execution."}}
{"t":{"$date":"2021-06-14T05:15:02.029+00:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn7","msg":"connection ended","attr":{"remote":"10.0.10.5:58242","connectionCount":3}}
{"t":{"$date":"2021-06-14T05:15:02.385+00:00"},"s":"I",  "c":"ROLLBACK", "id":21607,   "ctx":"BackgroundSync","msg":"Rollback common point","attr":{"commonPointOpTime":{"ts":{"$timestamp":{"t":1622300653,"i":1}},"t":35}}}
{"t":{"$date":"2021-06-14T05:15:02.385+00:00"},"s":"F",  "c":"ROLLBACK", "id":51121,   "ctx":"BackgroundSync","msg":"Common point must be at least stable timestamp","attr":{"commonPoint":{"$timestamp":{"t":1622300653,"i":1}},"stableTimestamp":{"$timestamp":{"t":1622300925,"i":1}}}}
{"t":{"$date":"2021-06-14T05:15:02.386+00:00"},"s":"F",  "c":"-",        "id":23091,   "ctx":"BackgroundSync","msg":"Fatal assertion","attr":{"msgid":51121,"file":"src\\mongo\\db\\repl\\rollback_impl.cpp","line":1035}}
{"t":{"$date":"2021-06-14T05:15:02.386+00:00"},"s":"F",  "c":"-",        "id":23092,   "ctx":"BackgroundSync","msg":"\n\n***aborting after fassert() failure\n\n"}
> 
> rs0:PRIMARY> rs.status()
> {
>     "set" : "rs0",
>     "date" : ISODate("2021-06-14T05:16:08.838Z"),
>     "myState" : 1,
>     "term" : NumberLong(38),
>     "syncSourceHost" : "",
>     "syncSourceId" : -1,
>     "heartbeatIntervalMillis" : NumberLong(2000),
>     "majorityVoteCount" : 2,
>     "writeMajorityCount" : 2,
>     "votingMembersCount" : 3,
>     "writableVotingMembersCount" : 2,
>     "optimes" : {
>             "lastCommittedOpTime" : {
>                     "ts" : Timestamp(0, 0),
>                     "t" : NumberLong(-1)
>             },
>             "lastCommittedWallTime" : ISODate("1970-01-01T00:00:00Z"),
>             "appliedOpTime" : {
>                     "ts" : Timestamp(1623647760, 1),
>                     "t" : NumberLong(38)
>             },
>             "durableOpTime" : {
>                     "ts" : Timestamp(1623647760, 1),
>                     "t" : NumberLong(38)
>             },
>             "lastAppliedWallTime" : ISODate("2021-06-14T05:16:00.082Z"),
>             "lastDurableWallTime" : ISODate("2021-06-14T05:16:00.082Z")
>     },
>     "lastStableRecoveryTimestamp" : Timestamp(1622300653, 1),
>     "electionCandidateMetrics" : {
>             "lastElectionReason" : "electionTimeout",
>             "lastElectionDate" : ISODate("2021-05-29T15:18:34.315Z"),
>             "electionTerm" : NumberLong(38),
>             "lastCommittedOpTimeAtElection" : {
>                     "ts" : Timestamp(0, 0),
>                     "t" : NumberLong(-1)
>             },
>             "lastSeenOpTimeAtElection" : {
>                     "ts" : Timestamp(1622300653, 1),
>                     "t" : NumberLong(35)
>             },
>             "numVotesNeeded" : 2,
>             "priorityAtElection" : 1,
>             "electionTimeoutMillis" : NumberLong(10000),
>             "numCatchUpOps" : NumberLong(0),
>             "newTermStartDate" : ISODate("2021-05-29T15:18:35.322Z")
>     },
>     "members" : [
>             {
>                     "_id" : 9,
>                     "name" : "mongo-ww6-clone:27018",
>                     "health" : 0,
>                     "state" : 8,
>                     "stateStr" : "(not reachable/healthy)",
>                     "uptime" : 0,
>                     "optime" : {
>                             "ts" : Timestamp(0, 0),
>                             "t" : NumberLong(-1)
>                     },
>                     "optimeDurable" : {
>                             "ts" : Timestamp(0, 0),
>                             "t" : NumberLong(-1)
>                     },
>                     "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
>                     "optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
>                     "lastHeartbeat" : ISODate("2021-06-14T05:16:05.955Z"),
>                     "lastHeartbeatRecv" : ISODate("2021-06-14T05:15:01.882Z"),
>                     "pingMs" : NumberLong(0),
>                     "lastHeartbeatMessage" : "Error connecting to mongo-ww6-clone:27018 (10.0.10.5:27018) :: caused by :: No connection could be made because the target machine actively refused it.",
>                     "syncSourceHost" : "",
>                     "syncSourceId" : -1,
>                     "infoMessage" : "",
>                     "configVersion" : 483019,
>                     "configTerm" : 38
>             },
>             {
>                     "_id" : 10,
>                     "name" : "mongo-ww6-clone:27019",
>                     "health" : 1,
>                     "state" : 7,
>                     "stateStr" : "ARBITER",
>                     "uptime" : 1346256,
>                     "lastHeartbeat" : ISODate("2021-06-14T05:16:07.513Z"),
>                     "lastHeartbeatRecv" : ISODate("2021-06-14T05:16:07.513Z"),
>                     "pingMs" : NumberLong(0),
>                     "lastHeartbeatMessage" : "",
>                     "syncSourceHost" : "",
>                     "syncSourceId" : -1,
>                     "infoMessage" : "",
>                     "configVersion" : 483019,
>                     "configTerm" : 38
>             },
>             {
>                     "_id" : 11,
>                     "name" : "mongo-ww6-clone:27017",
>                     "health" : 1,
>                     "state" : 1,
>                     "stateStr" : "PRIMARY",
>                     "uptime" : 1346320,
>                     "optime" : {
>                             "ts" : Timestamp(1623647760, 1),
>                             "t" : NumberLong(38)
>                     },
>                     "optimeDate" : ISODate("2021-06-14T05:16:00Z"),
>                     "syncSourceHost" : "",
>                     "syncSourceId" : -1,
>                     "infoMessage" : "",
>                     "electionTime" : Timestamp(1622301514, 1),
>                     "electionDate" : ISODate("2021-05-29T15:18:34Z"),
>                     "configVersion" : 483019,
>                     "configTerm" : 38,
>                     "self" : true,
>                     "lastHeartbeatMessage" : ""
>             }
>     ],
>     "ok" : 1,
>     "$clusterTime" : {
>             "clusterTime" : Timestamp(1623647760, 1),
>             "signature" : {
>                     "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
>                     "keyId" : NumberLong(0)
>             }
>     },
>     "operationTime" : Timestamp(1623647760, 1)
> }
> rs0:PRIMARY>