Hi,
Our setup details are as mentioned below.
1) Opensource on prem-K8s cluster with 3 Masters + 3 Workers.
2) Master and worker are same baremetal servers.
3) We are using longhorn storage provider.
4) We are using Mongodb community edition with 3 replicas which are spread across 3 worker nodes.
5) We are using Mongod operator to bring up the mongo cluster.
6) Mongo version details are as below
* mongod container version : mongodb-community-server:8.0.0-ubi8
* mongodb-agent container: mongodb-agent-ubi:107.0.7.8596-1
What action led to problem ?
Performed one of the worker node warm reboot using IDRAC.
What is the issue seen?
After node is rebooted , one of the mongod pod gets into crashloopback state. This pod never recovers.
Logs ?
{“t”:{“$date”:“2025-05-20T13:20:12.569+00:00”},“s”:“F”, “c”:“CONTROL”, “id”:6384300, “ctx”:“initandlisten”,“msg”:“Writing fatal message”,“attr”:{“message”:“Got signal: 11 (Segmentation fault).\n”}}
51 {“t”:{“$date”:“2025-05-20T13:20:12.616+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31380, “ctx”:“initandlisten”,“msg”:“BACKTRACE”,“attr”:
{“bt”:{“backtrace”:[{“a”:“5D8453788E18”,“b”:“5D844A968000”,“o”:“8E20 E18”,“s”:“_ZN5mongo15printStackTraceEv”,“C”:“mongo::printStackTrace()”,“s+”:“38”},
{“a”:“5D84537838F9”,“b”:“5D844A968000”,“o”:“8E1B8F9”,“s”:“abruptQuitWithAddrSignal”,“s+”:“139”},
{“a”:“7972CB8FD990”,“b”: “7972CB8EB000”,“o”:“12990”,“s”:“funlockfile”,“s+”:“50”},
{“a”:“7972CB536F41”,“b”:“7972CB515000”,“o”:“21F41”,“s”:“abort”,“s+”:“203”},
{“a”:“5D8453774791”,“b”:“5D844A968000”,“o”:“8E0C791”,“s”:“_ZN5mongo12_G LOBAL__N_19callAbortEv”,“C”:“mongo::(anonymous namespace)::callAbort()”,“s+”:“1B”},
{“a”:“5D84537759D3”,“b”:“5D844A968000”,“o”:“8E0D9D3”,“s”:“_ZN5mongo14fassert_detail6failedENS_14SourceLocationENS0_5Msg IdE”,
“C”:“mongo::fassert_detail::failed(mongo::SourceLocation, mongo::fassert_detail::MsgId)”,“s+”:“165”},
{“a”:“5D84506E53BF”,“b”:“5D844A968000”,“o”:“5D7D3BF”,“s”:“_ZN5mongo12_GLOBAL__N_141mdb_handle_er ror_with_startup_suppressionEP18__wt_event_handlerP12__wt_sessioniPKc.cold”,
“C”:“mongo::(anonymous namespace)::mdb_handle_error_with_startup_suppression(__wt_event_handler*, __wt_session*, int, char con st*) [clone .cold]”,“s+”:“16”},
{“a”:“5D84508C13A7”,“b”:“5D844A968000”,“o”:“5F593A7”,“s”:“__eventv”,“s+”:“10D7”},
{“a”:“5D84508C1FC3”,“b”:“5D844A968000”,“o”:“5F59FC3”,“s”:“__wt_panic_func”,“s+”:“154”},
{“a “:“5D845080E70B”,“b”:“5D844A968000”,“o”:“5EA670B”,“s”:”__wti_turtle_read.cold”,“s+”:“58”},
{“a”:“5D845080CEC9”,“b”:“5D844A968000”,“o”:“5EA4EC9”,“s”:“_wt_turtle_validate_version",“s+”:“179”},
{“a”:“5D8450 6FD637”,“b”:“5D844A968000”,“o”:“5D95637”,“s”:“wiredtiger_open”,“s+”:“2C87”},
{“a”:“5D8450691E41”,“b”:“5D844A968000”,“o”:“5D29E41”,“s”:"ZN5mongo18WiredTigerKVEngine15_openWiredTigerERKNSt7__cxx1112basic stringIcSt11char_traitsIcESaIcEEES8”,
“C”:“mongo::WiredTigerKVEngine::_openWiredTiger(std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, std::__cxx11::basic_string<c har, std::char_traits, std::allocator > const&)”,“s+”:“A1”},{“a”:“5D845069A89B”,“b”:“5D844A968000”,“o”:“5D3289B”,“s”:“_ZN5mongo18WiredTigerKVEngineC2ERKNSt7__cxx1112basic_stringIcSt11char_tr aitsIcESaIcEEES8_PNS_11ClockSourceES8_mmbb”,“C”:“mongo::WiredTigerKVEngine::WiredTigerKVEngine(std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, std::__cxx11::basic _string<char, std::char_traits, std::allocator > const&, mongo::ClockSource*, std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, unsigned long, unsigned long, bool, bool)”,“s+”:“B8B”},
{“a”:“5D845065A374”,“b”:“5D844A968000”,“o”:“5CF2374”,“s”:“_ZNK5mongo12_GLOBAL__N_117WiredTigerFactory6createEPNS_16OperationContextERKNS_19StorageGlobalParamsEPKNS_21Stora geEngineLockFileE”,“C”:“mongo::(anonymous namespace)::WiredTigerFactory::create(mongo::OperationContext*, mongo::StorageGlobalParams const&, mongo::StorageEngineLockFile const*)
const”,“s+”:“154”},{“a”: “5D8450A02141”,“b”:“5D844A968000”,“o”:“609A141”,“s”:“_ZN5mongo23initializeStorageEngineEPNS_16OperationContextENS_22StorageEngineInitFlagsEPNS_14BSONObjBuilderE”,“C”:“mongo::initializeStorageEngine(mong o::OperationContext*, mongo::StorageEngineInitFlags, mongo::BSONObjBuilder*)”,“s+”:“441”},{“a”:“5D844EFD71AD”,“b”:“5D844A968000”,“o”:“466F1AD”,“s”:“_ZN5mongo12_GLOBAL__N_114_initAndListenEPNS_14ServiceC ontextEi.constprop.0”,“C”:“mongo::(anonymous namespace)::initAndListen(mongo::ServiceContext*, int) [clone .constprop.0]",“s+”:“6FD”},{“a”:“5D844EFDB232”,“b”:“5D844A968000”,“o”:“4673232”,“s”:"ZN5mongo 11mongod_mainEiPPc",“C”:“mongo::mongod_main(int, char**)”,“s+”:“952”},
{“a”:“5D844EFD0549”,“b”:“5D844A968000”,“o”:“4668549”,“s”:“main”,“s+”:“9”},
{“a”:“7972CB54F7E5”,“b”:“7972CB515000”,“o”:“3A7E5”,“s”:" libc_start_main”,“s+”:“E5”},
{“a”:“5D844EFD042E”,“b”:“5D844A968000”,“o”:“466842E”,“s”:“_start”,“s+”:“2E”}],
“processInfo”:{“mongodbVersion”:“8.0.0”,“gitVersion”:“d7cd03b239ac39a3c7d63f7145e91aca36f93db6”,
“compiledModules”:,“uname”:{“sysname”:“Linux”,“release”:“6.8.0-59-generic”,“version”:“#61~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Tue Apr 15 17:03:15 UTC 2”,“machine”:“x86_64”},“somap”:[{“b”:“5D844A968000” ,“elfType”:3,“buildId”:“C1DAB2FBBBD2123D”},{“b”:“7972CB8EB000”,“path”:“/lib64/libpthread.so.0”,“elfType”:3,“buildId”:“FD0441CF253F643CA977DDFBC46257EDDAF544AE”},{“b”:“7972CB515000”,“path”:“/lib64/libc.s o.6”,“elfType”:3,“buildId”:“889235A2805B8308B2D0274921BBE1890E9A1986”}]}}},“tags”:}