Hello! i have a Replica set which contains of 3 nodes
but my 3rd replica accidentally stop working and if i type systemctl start mongod it says that
systemctl start mongod
Job for mongod.service failed because a timeout was exceeded. See "systemctl status mongod.service" and "journalctl -xe" for details.
and here is mongod logs
{"t":{"$date":"2022-09-06T16:28:30.241+06:00"},"s":"I", "c":"CONTROL", "id":20698, "ctx":"main","msg":"***** SERVER RESTARTED *****"}
{"t":{"$date":"2022-09-06T16:28:30.245+06:00"},"s":"I", "c":"CONTROL", "id":23285, "ctx":"main","msg":"Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify –sslDisabledProtocols 'none'"}
{"t":{"$date":"2022-09-06T16:28:30.271+06:00"},"s":"W", "c":"ASIO", "id":22601, "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
{"t":{"$date":"2022-09-06T16:28:30.271+06:00"},"s":"I", "c":"NETWORK", "id":4648601, "ctx":"main","msg":"Implicit TCP FastOpen unavailable. If TCP FastOpen is required, set tcpFastOpenServer, tcpFastOpenClient, and tcpFastOpenQueueSize."}
{"t":{"$date":"2022-09-06T16:28:30.271+06:00"},"s":"W", "c":"ASIO", "id":22601, "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
{"t":{"$date":"2022-09-06T16:28:30.271+06:00"},"s":"I", "c":"STORAGE", "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":7151,"port":27017,"dbPath":"/var/lib/mongo","architecture":"64-bit","host":"facetech-prod-mongo01-uv03.fortebank.com"}}
{"t":{"$date":"2022-09-06T16:28:30.271+06:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"4.4.3","gitVersion":"913d6b62acfbb344dde1b116f4161360acd8fd13","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":[],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}}
{"t":{"$date":"2022-09-06T16:28:30.272+06:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.42.2.el7.x86_64"}}}
{"t":{"$date":"2022-09-06T16:28:30.272+06:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/etc/mongod.conf","net":{"bindIp":"10.0.225.235","port":27017},"processManagement":{"fork":true,"pidFilePath":"/var/run/mongodb/mongod.pid","timeZoneInfo":"/usr/share/zoneinfo"},"replication":{"replSetName":"prod-facetech"},"storage":{"dbPath":"/var/lib/mongo","journal":{"enabled":true}},"systemLog":{"destination":"file","logAppend":true,"path":"/var/log/mongodb/mongod.log"}}}}
{"t":{"$date":"2022-09-06T16:28:30.273+06:00"},"s":"W", "c":"STORAGE", "id":22271, "ctx":"initandlisten","msg":"Detected unclean shutdown - Lock file is not empty","attr":{"lockFile":"/var/lib/mongo/mongod.lock"}}
{"t":{"$date":"2022-09-06T16:28:30.273+06:00"},"s":"I", "c":"STORAGE", "id":22270, "ctx":"initandlisten","msg":"Storage engine to use detected by data files","attr":{"dbpath":"/var/lib/mongo","storageEngine":"wiredTiger"}}
{"t":{"$date":"2022-09-06T16:28:30.273+06:00"},"s":"W", "c":"STORAGE", "id":22302, "ctx":"initandlisten","msg":"Recovering data from the last clean checkpoint."}
{"t":{"$date":"2022-09-06T16:28:30.273+06:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=3398M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],"}}
{"t":{"$date":"2022-09-06T16:28:30.879+06:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1662460110:879327][7151:0x7f63aed90bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 37908 through 37909"}}
{"t":{"$date":"2022-09-06T16:28:30.975+06:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1662460110:975619][7151:0x7f63aed90bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 37909 through 37909"}}
{"t":{"$date":"2022-09-06T16:28:31.108+06:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1662460111:108841][7151:0x7f63aed90bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 37908/256 to 37909/256"}}
{"t":{"$date":"2022-09-06T16:28:31.241+06:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1662460111:241647][7151:0x7f63aed90bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 37908 through 37909"}}
{"t":{"$date":"2022-09-06T16:28:31.331+06:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1662460111:331583][7151:0x7f63aed90bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 37909 through 37909"}}
{"t":{"$date":"2022-09-06T16:28:31.395+06:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1662460111:395447][7151:0x7f63aed90bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (1659204188, 1)"}}
{"t":{"$date":"2022-09-06T16:28:31.395+06:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1662460111:395512][7151:0x7f63aed90bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (1659204183, 1)"}}
{"t":{"$date":"2022-09-06T16:28:32.098+06:00"},"s":"I", "c":"STORAGE", "id":4795906, "ctx":"initandlisten","msg":"WiredTiger opened","attr":{"durationMillis":1825}}
{"t":{"$date":"2022-09-06T16:28:32.098+06:00"},"s":"I", "c":"RECOVERY", "id":23987, "ctx":"initandlisten","msg":"WiredTiger recoveryTimestamp","attr":{"recoveryTimestamp":{"$timestamp":{"t":1659204188,"i":1}}}}
{"t":{"$date":"2022-09-06T16:28:32.099+06:00"},"s":"I", "c":"STORAGE", "id":4366408, "ctx":"initandlisten","msg":"No table logging settings modifications are required for existing WiredTiger tables","attr":{"loggingEnabled":false}}
{"t":{"$date":"2022-09-06T16:28:32.103+06:00"},"s":"I", "c":"STORAGE", "id":22383, "ctx":"initandlisten","msg":"The size storer reports that the oplog contains","attr":{"numRecords":205531,"dataSize":53787115711}}
{"t":{"$date":"2022-09-06T16:28:32.103+06:00"},"s":"I", "c":"STORAGE", "id":22386, "ctx":"initandlisten","msg":"Sampling the oplog to determine where to place markers for truncation"}
{"t":{"$date":"2022-09-06T16:28:32.107+06:00"},"s":"I", "c":"STORAGE", "id":22389, "ctx":"initandlisten","msg":"Sampling from the oplog to determine where to place markers for truncation","attr":{"from":{"$timestamp":{"t":1658914933,"i":1}},"to":{"$timestamp":{"t":1659204198,"i":1}}}}
{"t":{"$date":"2022-09-06T16:28:32.107+06:00"},"s":"I", "c":"STORAGE", "id":22390, "ctx":"initandlisten","msg":"Taking samples and assuming each oplog section contains","attr":{"numSamples":1001,"containsNumRecords":2052,"containsNumBytes":537004935}}
{"t":{"$date":"2022-09-06T16:28:42.159+06:00"},"s":"I", "c":"STORAGE", "id":22392, "ctx":"initandlisten","msg":"Oplog sampling progress","attr":{"completed":53,"total":1001}}
{"t":{"$date":"2022-09-06T16:28:52.198+06:00"},"s":"I", "c":"STORAGE", "id":22392, "ctx":"initandlisten","msg":"Oplog sampling progress","attr":{"completed":103,"total":1001}}
{"t":{"$date":"2022-09-06T16:29:02.199+06:00"},"s":"I", "c":"STORAGE", "id":22392, "ctx":"initandlisten","msg":"Oplog sampling progress","attr":{"completed":157,"total":1001}}
{"t":{"$date":"2022-09-06T16:29:12.200+06:00"},"s":"I", "c":"STORAGE", "id":22392, "ctx":"initandlisten","msg":"Oplog sampling progress","attr":{"completed":219,"total":1001}}
{"t":{"$date":"2022-09-06T16:29:22.450+06:00"},"s":"I", "c":"STORAGE", "id":22392, "ctx":"initandlisten","msg":"Oplog sampling progress","attr":{"completed":277,"total":1001}}
{"t":{"$date":"2022-09-06T16:29:32.521+06:00"},"s":"I", "c":"STORAGE", "id":22392, "ctx":"initandlisten","msg":"Oplog sampling progress","attr":{"completed":352,"total":1001}}
{"t":{"$date":"2022-09-06T16:29:42.660+06:00"},"s":"I", "c":"STORAGE", "id":22392, "ctx":"initandlisten","msg":"Oplog sampling progress","attr":{"completed":422,"total":1001}}
{"t":{"$date":"2022-09-06T16:29:52.741+06:00"},"s":"I", "c":"STORAGE", "id":22392, "ctx":"initandlisten","msg":"Oplog sampling progress","attr":{"completed":500,"total":1001}}
{"t":{"$date":"2022-09-06T16:30:00.241+06:00"},"s":"I", "c":"CONTROL", "id":23377, "ctx":"SignalHandler","msg":"Received signal","attr":{"signal":15,"error":"Terminated"}}
{"t":{"$date":"2022-09-06T16:30:00.241+06:00"},"s":"I", "c":"CONTROL", "id":23378, "ctx":"SignalHandler","msg":"Signal was sent by kill(2)","attr":{"pid":1,"uid":0}}
{"t":{"$date":"2022-09-06T16:30:00.245+06:00"},"s":"I", "c":"CONTROL", "id":23381, "ctx":"SignalHandler","msg":"will terminate after current cmd ends"}
{"t":{"$date":"2022-09-06T16:30:00.261+06:00"},"s":"I", "c":"REPL", "id":4784900, "ctx":"SignalHandler","msg":"Stepping down the ReplicationCoordinator for shutdown","attr":{"waitTimeMillis":10000}}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"COMMAND", "id":4784901, "ctx":"SignalHandler","msg":"Shutting down the MirrorMaestro"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"SHARDING", "id":4784902, "ctx":"SignalHandler","msg":"Shutting down the WaitForMajorityService"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"NETWORK", "id":20562, "ctx":"SignalHandler","msg":"Shutdown: going to close listening sockets"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"NETWORK", "id":4784905, "ctx":"SignalHandler","msg":"Shutting down the global connection pool"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"STORAGE", "id":4784906, "ctx":"SignalHandler","msg":"Shutting down the FlowControlTicketholder"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"-", "id":20520, "ctx":"SignalHandler","msg":"Stopping further Flow Control ticket acquisitions."}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"REPL", "id":4784907, "ctx":"SignalHandler","msg":"Shutting down the replica set node executor"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"NETWORK", "id":4784918, "ctx":"SignalHandler","msg":"Shutting down the ReplicaSetMonitor"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"SHARDING", "id":4784921, "ctx":"SignalHandler","msg":"Shutting down the MigrationUtilExecutor"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"CONTROL", "id":4784925, "ctx":"SignalHandler","msg":"Shutting down free monitoring"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"STORAGE", "id":4784927, "ctx":"SignalHandler","msg":"Shutting down the HealthLog"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"STORAGE", "id":4784929, "ctx":"SignalHandler","msg":"Acquiring the global lock for shutdown"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"-", "id":4784931, "ctx":"SignalHandler","msg":"Dropping the scope cache for shutdown"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"FTDC", "id":4784926, "ctx":"SignalHandler","msg":"Shutting down full-time data capture"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"CONTROL", "id":20565, "ctx":"SignalHandler","msg":"Now exiting"}
{"t":{"$date":"2022-09-06T16:30:00.305+06:00"},"s":"I", "c":"CONTROL", "id":23138, "ctx":"SignalHandler","msg":"Shutting down","attr":{"exitCode":0}}