MongoDB config server and share0 could not start

My mongodb sharding config server and share-0 could not start, maybe cause a unscheduled power off, and I try to repair it, but it still not work. Could same one please help me to resolve this.

Here is config server start log.

{"t":{"$date":"2022-06-06T06:08:04.545+00:00"},"s":"I",  "c":"SHARDING", "id":22649,   "ctx":"thread24","msg":"Creating distributed lock ping thread","attr":{"processId":"config","pingIntervalMillis":30000}}
{"t":{"$date":"2022-06-06T06:08:04.545+00:00"},"s":"I",  "c":"REPL",     "id":6015317, "ctx":"initandlisten","msg":"Setting new configuration state","attr":{"newState":"ConfigStartingUp","oldState":"ConfigPreStart"}}
{"t":{"$date":"2022-06-06T06:08:04.546+00:00"},"s":"I",  "c":"REPL",     "id":4280500, "ctx":"initandlisten","msg":"Attempting to create internal replication collections"}
{"t":{"$date":"2022-06-06T06:08:04.547+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":-31802,"message":"[1654495684:547876][1:0x7f3f6a96ac80], file:collection-2-5837153500946542875.wt, WT_SESSION.open_cursor: __posix_file_read, 428: /data/db/collection-2-5837153500946542875.wt: handle-read: pread: failed to read 4096 bytes at offset 32768: WT_ERROR: non-specific WiredTiger error"}}
{"t":{"$date":"2022-06-06T06:08:04.548+00:00"},"s":"F",  "c":"STORAGE",  "id":50882,   "ctx":"initandlisten","msg":"Failed to open WiredTiger cursor. This may be due to data corruption","attr":{"uri":"table:collection-2-5837153500946542875","config":"","error":{"code":8,"codeName":"UnknownError","errmsg":"-31802: WT_ERROR: non-specific WiredTiger error"},"message":"Please read the documentation for starting MongoDB with --repair here: http://dochub.mongodb.org/core/repair"}}
{"t":{"$date":"2022-06-06T06:08:04.548+00:00"},"s":"F",  "c":"-",        "id":23091,   "ctx":"initandlisten","msg":"Fatal assertion","attr":{"msgid":50882,"file":"src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp","line":109}}
{"t":{"$date":"2022-06-06T06:08:04.548+00:00"},"s":"F",  "c":"-",        "id":23092,   "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"}

And here is share-0 start log

{"t":{"$date":"2022-06-06T06:23:50.220+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":-31802,"message":"[1654496630:220679][1:0x7f0d18c36c80], file:WiredTiger.wt, connection: __wt_block_read_off, 309: WiredTiger.wt: fatal read error: WT_ERROR: non-specific WiredTiger error"}}
{"t":{"$date":"2022-06-06T06:23:50.220+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":-31804,"message":"[1654496630:220691][1:0x7f0d18c36c80], file:WiredTiger.wt, connection: __wt_block_read_off, 309: the process must exit and restart: WT_PANIC: WiredTiger library panic"}}
{"t":{"$date":"2022-06-06T06:23:50.220+00:00"},"s":"F",  "c":"-",        "id":23089,   "ctx":"initandlisten","msg":"Fatal assertion","attr":{"msgid":50853,"file":"src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp","line":538}}
{"t":{"$date":"2022-06-06T06:23:50.220+00:00"},"s":"F",  "c":"-",        "id":23090,   "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"}
{"t":{"$date":"2022-06-06T06:23:50.220+00:00"},"s":"F",  "c":"CONTROL",  "id":4757800, "ctx":"initandlisten","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31380,   "ctx":"initandlisten","msg":"BACKTRACE","attr":{"bt":{"backtrace":[{"a":"563853EB7405","b":"563850008000","o":"3EAF405","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.357","s+":"215"},{"a":"563853EB9E99","b":"563850008000","o":"3EB1E99","s":"_ZN5mongo15printStackTraceEv","s+":"29"},{"a":"563853EB24D6","b":"563850008000","o":"3EAA4D6","s":"abruptQuit","s+":"66"},{"a":"7F0D19AF43C0","b":"7F0D19AE0000","o":"143C0","s":"funlockfile","s+":"60"},{"a":"7F0D1993103B","b":"7F0D198EE000","o":"4303B","s":"gsignal","s+":"CB"},{"a":"7F0D19910859","b":"7F0D198EE000","o":"22859","s":"abort","s+":"12B"},{"a":"5638513CF06B","b":"563850008000","o":"13C706B","s":"_ZN5mongo25fassertFailedWithLocationEiPKcj","s+":"F6"},{"a":"563850EC1522","b":"563850008000","o":"EB9522","s":"_ZN5mongo12_GLOBAL__N_141mdb_handle_error_with_startup_suppressionEP18__wt_event_handlerP12__wt_sessioniPKc.cold.1216","s+":"16"},{"a":"5638516D1F03","b":"563850008000","o":"16C9F03","s":"__eventv","s+":"403"},{"a":"563850ED3E84","b":"563850008000","o":"ECBE84","s":"__wt_panic_func","s+":"114"},{"a":"563850EE2590","b":"563850008000","o":"EDA590","s":"__wt_block_read_off.cold.5","s+":"D3"},{"a":"5638517FC966","b":"563850008000","o":"17F4966","s":"__wt_block_extlist_read","s+":"96"},{"a":"5638517FCE8B","b":"563850008000","o":"17F4E8B","s":"__wt_block_extlist_read_avail","s+":"2B"},{"a":"563851806560","b":"563850008000","o":"17FE560","s":"__wt_block_checkpoint_load","s+":"1E0"},{"a":"5638517FE007","b":"563850008000","o":"17F6007","s":"__bm_checkpoint_load","s+":"37"},{"a":"56385172A662","b":"563850008000","o":"1722662","s":"__wt_btree_open","s+":"D62"},{"a":"563851650412","b":"563850008000","o":"1648412","s":"__wt_conn_dhandle_open","s+":"8D2"},{"a":"5638516D07D9","b":"563850008000","o":"16C87D9","s":"__wt_session_get_dhandle","s+":"E9"},{"a":"5638516D0E82","b":"563850008000","o":"16C8E82","s":"__wt_session_get_
dhandle","s+":"792"},{"a":"5638516D1184","b":"563850008000","o":"16C9184","s":"__wt_session_get_btree_ckpt","s+":"154"},{"a":"56385166848A","b":"563850008000","o":"166048A","s":"__wt_curfile_open","s+":"5A"},{"a":"5638516CB7BB","b":"563850008000","o":"16C37BB","s":"__session_open_cursor_int","s+":"2DB"},{"a":"5638516CB478","b":"563850008000","o":"16C3478","s":"__wt_open_cursor","s+":"58"},{"a":"563851696E3E","b":"563850008000","o":"168EE3E","s":"__wt_metadata_cursor_open","s+":"6E"},{"a":"563851696F3B","b":"563850008000","o":"168EF3B","s":"__wt_metadata_cursor","s+":"4B"},{"a":"56385164CD97","b":"563850008000","o":"1644D97","s":"wiredtiger_open","s+":"28C7"},{"a":"5638515F8239","b":"563850008000","o":"15F0239","s":"_ZN5mongo18WiredTigerKVEngine15_openWiredTigerERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_","s+":"B9"},{"a":"563851603738","b":"563850008000","o":"15FB738","s":"_ZN5mongo18WiredTigerKVEngineC2ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_PNS_11ClockSourceES8_mmbbbb","s+":"1138"},{"a":"5638515DA121","b":"563850008000","o":"15D2121","s":"_ZNK5mongo12_GLOBAL__N_117WiredTigerFactory6createEPNS_16OperationContextERKNS_19StorageGlobalParamsEPKNS_21StorageEngineLockFileE","s+":"171"},{"a":"5638523BC289","b":"563850008000","o":"23B4289","s":"_ZN5mongo23initializeStorageEngineEPNS_16OperationContextENS_22StorageEngineInitFlagsE","s+":"419"},{"a":"56385154382F","b":"563850008000","o":"153B82F","s":"_ZN5mongo12_GLOBAL__N_114_initAndListenEPNS_14ServiceContextEi.isra.1896","s+":"47F"},{"a":"5638515461AF","b":"563850008000","o":"153E1AF","s":"_ZN5mongo11mongod_mainEiPPc","s+":"CDF"},{"a":"5638513E237E","b":"563850008000","o":"13DA37E","s":"main","s+":"E"},{"a":"7F0D199120B3","b":"7F0D198EE000","o":"240B3","s":"__libc_start_main","s+":"F3"},{"a":"5638515406AE","b":"563850008000","o":"15386AE","s":"_start","s+":"2E"}],"processInfo":{"mongodbVersion":"5.0.7","gitVersion":"b977129dc70eed766cbee7e412d901ee213acbda","compiledModules":[],"uname":{"sysname":"Linux","release":"5.4.0-113-generic","v
ersion":"#127-Ubuntu SMP Wed May 18 14:30:56 UTC 2022","machine":"x86_64"},"somap":[{"b":"563850008000","elfType":3,"buildId":"FD1987341101CD5988745C190BB51D0DA505C680"},{"b":"7F0D19AE0000","path":"/lib/x86_64-linux-gnu/libpthread.so.0","elfType":3,"buildId":"F0983025F0E0F327A6DA752FF4FFA675E0BE393F"},{"b":"7F0D198EE000","path":"/lib/x86_64-linux-gnu/libc.so.6","elfType":3,"buildId":"9FDB74E7B217D06C93172A8243F8547F947EE6D1"}]}}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"563853EB7405","b":"563850008000","o":"3EAF405","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.357","s+":"215"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"563853EB9E99","b":"563850008000","o":"3EB1E99","s":"_ZN5mongo15printStackTraceEv","s+":"29"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"563853EB24D6","b":"563850008000","o":"3EAA4D6","s":"abruptQuit","s+":"66"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"7F0D19AF43C0","b":"7F0D19AE0000","o":"143C0","s":"funlockfile","s+":"60"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"7F0D1993103B","b":"7F0D198EE000","o":"4303B","s":"gsignal","s+":"CB"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"7F0D19910859","b":"7F0D198EE000","o":"22859","s":"abort","s+":"12B"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638513CF06B","b":"563850008000","o":"13C706B","s":"_ZN5mongo25fassertFailedWithLocationEiPKcj","s+":"F6"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"563850EC1522","b":"563850008000","o":"EB9522","s":"_ZN5mongo12_GLOBAL__N_141mdb_handle_error_with_startup_suppressionEP18__wt_event_handlerP12__wt_sessioniPKc.cold.1216","s+":"16"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638516D1F03","b":"563850008000","o":"16C9F03","s":"__eventv","s+":"403"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"563850ED3E84","b":"563850008000","o":"ECBE84","s":"__wt_panic_func","s+":"114"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"563850EE2590","b":"563850008000","o":"EDA590","s":"__wt_block_read_off.cold.5","s+":"D3"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638517FC966","b":"563850008000","o":"17F4966","s":"__wt_block_extlist_read","s+":"96"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638517FCE8B","b":"563850008000","o":"17F4E8B","s":"__wt_block_extlist_read_avail","s+":"2B"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"563851806560","b":"563850008000","o":"17FE560","s":"__wt_block_checkpoint_load","s+":"1E0"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638517FE007","b":"563850008000","o":"17F6007","s":"__bm_checkpoint_load","s+":"37"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"56385172A662","b":"563850008000","o":"1722662","s":"__wt_btree_open","s+":"D62"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"563851650412","b":"563850008000","o":"1648412","s":"__wt_conn_dhandle_open","s+":"8D2"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638516D07D9","b":"563850008000","o":"16C87D9","s":"__wt_session_get_dhandle","s+":"E9"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638516D0E82","b":"563850008000","o":"16C8E82","s":"__wt_session_get_dhandle","s+":"792"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638516D1184","b":"563850008000","o":"16C9184","s":"__wt_session_get_btree_ckpt","s+":"154"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"56385166848A","b":"563850008000","o":"166048A","s":"__wt_curfile_open","s+":"5A"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638516CB7BB","b":"563850008000","o":"16C37BB","s":"__session_open_cursor_int","s+":"2DB"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638516CB478","b":"563850008000","o":"16C3478","s":"__wt_open_cursor","s+":"58"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"563851696E3E","b":"563850008000","o":"168EE3E","s":"__wt_metadata_cursor_open","s+":"6E"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"563851696F3B","b":"563850008000","o":"168EF3B","s":"__wt_metadata_cursor","s+":"4B"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"56385164CD97","b":"563850008000","o":"1644D97","s":"wiredtiger_open","s+":"28C7"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638515F8239","b":"563850008000","o":"15F0239","s":"_ZN5mongo18WiredTigerKVEngine15_openWiredTigerERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_","s+":"B9"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"563851603738","b":"563850008000","o":"15FB738","s":"_ZN5mongo18WiredTigerKVEngineC2ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_PNS_11ClockSourceES8_mmbbbb","s+":"1138"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638515DA121","b":"563850008000","o":"15D2121","s":"_ZNK5mongo12_GLOBAL__N_117WiredTigerFactory6createEPNS_16OperationContextERKNS_19StorageGlobalParamsEPKNS_21StorageEngineLockFileE","s+":"171"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638523BC289","b":"563850008000","o":"23B4289","s":"_ZN5mongo23initializeStorageEngineEPNS_16OperationContextENS_22StorageEngineInitFlagsE","s+":"419"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"56385154382F","b":"563850008000","o":"153B82F","s":"_ZN5mongo12_GLOBAL__N_114_initAndListenEPNS_14ServiceContextEi.isra.1896","s+":"47F"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638515461AF","b":"563850008000","o":"153E1AF","s":"_ZN5mongo11mongod_mainEiPPc","s+":"CDF"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638513E237E","b":"563850008000","o":"13DA37E","s":"main","s+":"E"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"7F0D199120B3","b":"7F0D198EE000","o":"240B3","s":"__libc_start_main","s+":"F3"}}}
{"t":{"$date":"2022-06-06T06:23:50.369+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"initandlisten","msg":"Frame","attr":{"frame":{"a":"5638515406AE","b":"563850008000","o":"15386AE","s":"_start","s+":"2E"}}}

Hi @Alex_Zhou welcome to the community!

From the logs you posted, I believe the main issue is hardware. This is due to this line: failed to read 4096 bytes at offset 32768, meaning that WT tried to read a file, but failed. Power issues are definitely a possible source of corruption, but this could also be caused by bad hardware. Unfortunately, there’s not much anyone can do if the actual hardware is having issues.

The ideal resolution to this is to perform a clean resync from an unaffected node in the replica set. Alternatively, using mongod --repair might also work, but a clean resync is much preferable & safer.

In light of this event, I would recommend you to audit your remaining hardware and ensure that none of them are corrupt in this manner.

Best regards
Kevin