My mongodb server crashes often with this error, I am using MongoDB 7.0

MongoDB for Startups
, ,
#1

This happens 3-4 times every day. If I restart the server it is good for sometime. But then this error happens and server crashes. Then the container restarts.

What could be the root cause of this error ?

{"t":{"$date":"2023-09-04T06:22:05.513+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"Connection accepted","attr":{"remote":"10.100.0.13:56080","uuid":{"uuid":{"$uuid":"09c2c7ca-63b9-491b-a006-45c57e8a057d"}},"connectionId":375,"connectionCount":32}}
{"t":{"$date":"2023-09-04T06:22:05.513+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn375","msg":"client metadata","attr":{"remote":"10.100.0.13:56080","client":"conn375","doc":{"driver":{"name":"mongoc / ext-mongodb:PHP / PHPLIB ","version":"1.24.1 / 1.16.1 / 1.16.0 "},"os":{"type":"Linux","name":"Ubuntu","version":"22.04","architecture":"x86_64"},"platform":"PHP 8.1.2-1ubuntu2.13 cfg=0x03515620c9 posix=200809 stdc=201710 CC=GCC 11.3.0 CFLAGS=\"\" LDFLAGS=\"\""}}}
{"t":{"$date":"2023-09-04T06:22:05.514+00:00"},"s":"I",  "c":"NETWORK",  "id":6788700, "ctx":"conn375","msg":"Received first command on ingress connection since session start or auth handshake","attr":{"elapsedMillis":0}}
{"t":{"$date":"2023-09-04T06:22:05.569+00:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn374","msg":"Connection ended","attr":{"remote":"10.100.0.13:56078","uuid":{"uuid":{"$uuid":"c309384d-51fe-4dd5-8ca5-755e40098aaf"}},"connectionId":374,"connectionCount":31}}
{"t":{"$date":"2023-09-04T06:22:05.837+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"Connection accepted","attr":{"remote":"10.100.0.13:56092","uuid":{"uuid":{"$uuid":"ff59fcb5-221e-4b2c-bc6b-47d0b5fcd98e"}},"connectionId":376,"connectionCount":32}}
{"t":{"$date":"2023-09-04T06:22:05.837+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn376","msg":"client metadata","attr":{"remote":"10.100.0.13:56092","client":"conn376","doc":{"driver":{"name":"mongoc / ext-mongodb:PHP / PHPLIB ","version":"1.24.1 / 1.16.1 / 1.16.0 "},"os":{"type":"Linux","name":"Ubuntu","version":"22.04","architecture":"x86_64"},"platform":"PHP 8.1.2-1ubuntu2.13 cfg=0x03515620c9 posix=200809 stdc=201710 CC=GCC 11.3.0 CFLAGS=\"\" LDFLAGS=\"\""}}}
{"t":{"$date":"2023-09-04T06:22:05.840+00:00"},"s":"I",  "c":"NETWORK",  "id":6788700, "ctx":"conn376","msg":"Received first command on ingress connection since session start or auth handshake","attr":{"elapsedMillis":2}}
{"t":{"$date":"2023-09-04T06:22:05.892+00:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn376","msg":"Connection ended","attr":{"remote":"10.100.0.13:56092","uuid":{"uuid":{"$uuid":"ff59fcb5-221e-4b2c-bc6b-47d0b5fcd98e"}},"connectionId":376,"connectionCount":31}}
{"t":{"$date":"2023-09-04T06:22:05.892+00:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn375","msg":"Connection ended","attr":{"remote":"10.100.0.13:56080","uuid":{"uuid":{"$uuid":"09c2c7ca-63b9-491b-a006-45c57e8a057d"}},"connectionId":375,"connectionCount":30}}
{"t":{"$date":"2023-09-04T06:22:07.788+00:00"},"s":"I",  "c":"COMMAND",  "id":51803,   "ctx":"conn34","msg":"Slow query","attr":{"type":"command","ns":"selfmadelabs.$cmd","command":{"update":"labstats","ordered":true,"lsid":{"id":{"$uuid":"fa5453b3-980c-4e17-a8c6-46ae49513c04"}},"$db":"selfmadelabs"},"numYields":1372,"reslen":60,"locks":{"ParallelBatchWriterMode":{"acquireCount":{"r":5096}},"FeatureCompatibilityVersion":{"acquireCount":{"w":5096}},"ReplicationStateTransition":{"acquireCount":{"w":5096}},"Global":{"acquireCount":{"w":5096}},"Database":{"acquireCount":{"w":5096}},"Collection":{"acquireCount":{"w":5096}}},"flowControl":{"acquireCount":5096},"storage":{},"cpuNanos":3069647525,"remote":"10.100.0.12:59344","protocol":"op_msg","durationMillis":3115}}
{"t":{"$date":"2023-09-04T06:22:10.145+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"conn27","msg":"WiredTiger error message","attr":{"error":22,"message":{"ts_sec":1693808530,"ts_usec":145725,"thread":"1:0x7f2df5821640","session_name":"WT_SESSION.get_value","category":"WT_VERB_DEFAULT","category_id":9,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"__wt_txn_context_prepare_check:19:not permitted in a prepared transaction","error_str":"Invalid argument","error_code":22}}}
{"t":{"$date":"2023-09-04T06:22:10.145+00:00"},"s":"F",  "c":"ASSERT",   "id":23083,   "ctx":"conn27","msg":"Invariant failure","attr":{"expr":"c->get_value(c, &value)","error":"BadValue: 22: Invalid argument","file":"src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp","line":1975}}
{"t":{"$date":"2023-09-04T06:22:10.145+00:00"},"s":"F",  "c":"ASSERT",   "id":23084,   "ctx":"conn27","msg":"\n\n***aborting after invariant() failure\n\n"}
{"t":{"$date":"2023-09-04T06:22:10.145+00:00"},"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"conn27","msg":"Writing fatal message","attr":{"message":"\n"}}
{"t":{"$date":"2023-09-04T06:22:10.145+00:00"},"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"conn27","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}
{"t":{"$date":"2023-09-04T06:22:10.280+00:00"},"s":"I",  "c":"CONTROL",  "id":31380,   "ctx":"conn27","msg":"BACKTRACE","attr":{"bt":{"backtrace":[{"a":"5633C7E1AFB4","b":"5633C03FE000","o":"7A1CFB4","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_117getStackTraceImplERKNS1_7OptionsE.constprop.0","C":"mongo::stack_trace_detail::(anonymous namespace)::getStackTraceImpl(mongo::stack_trace_detail::(anonymous namespace)::Options const&) [clone .constprop.0]","s+":"224"},{"a":"5633C7E1CC68","b":"5633C03FE000","o":"7A1EC68","s":"_ZN5mongo15printStackTraceEv","C":"mongo::printStackTrace()","s+":"38"},{"a":"5633C7E177EA","b":"5633C03FE000","o":"7A197EA","s":"abruptQuit","s+":"6A"},{"a":"7F2E09AA8520","b":"7F2E09A66000","o":"42520","s":"__sigaction","s+":"50"},{"a":"7F2E09AFCA7C","b":"7F2E09A66000","o":"96A7C","s":"pthread_kill","s+":"12C"},{"a":"7F2E09AA8476","b":"7F2E09A66000","o":"42476","s":"raise","s+":"16"},{"a":"7F2E09A8E7F3","b":"7F2E09A66000","o":"287F3","s":"abort","s+":"D3"},{"a":"5633C7E09A47","b":"5633C03FE000","o":"7A0BA47","s":"_ZN5mongo12_GLOBAL__N_19callAbortEv","C":"mongo::(anonymous namespace)::callAbort()","s+":"1B"},{"a":"5633C7E0AF98","b":"5633C03FE000","o":"7A0CF98","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j","C":"mongo::invariantOKFailed(char const*, mongo::Status const&, char const*, unsigned int)","s+":"279"},{"a":"5633C4D159D5","b":"5633C03FE000","o":"49179D5","s":"_ZN5mongo31WiredTigerRecordStoreCursorBase4nextEv.cold","C":"mongo::WiredTigerRecordStoreCursorBase::next() [clone .cold]","s+":"2BB"},{"a":"5633C5D16C51","b":"5633C03FE000","o":"5918C51","s":"_ZN5mongo3sbe9ScanStage7getNextEv","C":"mongo::sbe::ScanStage::getNext()","s+":"2D1"},{"a":"5633C58A81E7","b":"5633C03FE000","o":"54AA1E7","s":"_ZN5mongo3sbe11FilterStageILb0ELb0EE7getNextEv","C":"mongo::sbe::FilterStage<false, false>::getNext()","s+":"57"},{"a":"5633C5C23915","b":"5633C03FE000","o":"5825915","s":"_ZN5mongo3sbe14LimitSkipStage7getNextEv","C":"mongo::sbe::LimitSkipStage::getNext()","s+":"D5"},{"a":"5633C584E2EC","b":"5633C03FE000","o":"54502EC","s":"_ZN5mongo13fetchNextImplINS_7BSONObjEEENS_3sbe9PlanStateEPNS2_9PlanStageEPNS2_5value12SlotAccessorES8_PT_PNS_8RecordIdEb","C":"mongo::sbe::PlanState mongo::fetchNextImpl<mongo::BSONObj>(mongo::sbe::PlanStage*, mongo::sbe::value::SlotAccessor*, mongo::sbe::value::SlotAccessor*, mongo::BSONObj*, mongo::RecordId*, bool)","s+":"4C"},{"a":"5633C584F62C","b":"5633C03FE000","o":"545162C","s":"_ZN5mongo15PlanExecutorSBE11getNextImplINS_7BSONObjEEENS_12PlanExecutor9ExecStateEPT_PNS_8RecordIdE","C":"mongo::PlanExecutor::ExecState mongo::PlanExecutorSBE::getNextImpl<mongo::BSONObj>(mongo::BSONObj*, mongo::RecordId*)","s+":"38C"},{"a":"5633C584A98B","b":"5633C03FE000","o":"544C98B","s":"_ZN5mongo15PlanExecutorSBE7getNextEPNS_7BSONObjEPNS_8RecordIdE","C":"mongo::PlanExecutorSBE::getNext(mongo::BSONObj*, mongo::RecordId*)","s+":"5B"},{"a":"5633C50322E0","b":"5633C03FE000","o":"4C342E0","s":"_ZN5mongo12_GLOBAL__N_17FindCmd10Invocation3runEPNS_16OperationContextEPNS_3rpc21ReplyBuilderInterfaceE","C":"mongo::(anonymous namespace)::FindCmd::Invocation::run(mongo::OperationContext*, mongo::rpc::ReplyBuilderInterface*)","s+":"E70"},{"a":"5633C747D9B0","b":"5633C03FE000","o":"707F9B0","s":"_ZN5mongo14CommandHelpers20runCommandInvocationEPNS_16OperationContextERKNS_12OpMsgRequestEPNS_17CommandInvocationEPNS_3rpc21ReplyBuilderInterfaceE","C":"mongo::CommandHelpers::runCommandInvocation(mongo::OperationContext*, mongo::OpMsgRequest const&, mongo::CommandInvocation*, mongo::rpc::ReplyBuilderInterface*)","s+":"60"},{"a":"5633C74818ED","b":"5633C03FE000","o":"70838ED","s":"_ZN5mongo14CommandHelpers20runCommandInvocationESt10shared_ptrINS_23RequestExecutionContextEES1_INS_17CommandInvocationEEb","C":"mongo::CommandHelpers::runCommandInvocation(std::shared_ptr<mongo::RequestExecutionContext>, std::shared_ptr<mongo::CommandInvocation>, bool)","s+":"CD"},{"a":"5633C3E8D3F0","b":"5633C03FE000","o":"3A8F3F0","s":"_ZN5mongo12_GLOBAL__N_120runCommandInvocationESt10shared_ptrINS_23RequestExecutionContextEES1_INS_17CommandInvocationEE","C":"mongo::(anonymous namespace)::runCommandInvocation(std::shared_ptr<mongo::RequestExecutionContext>, std::shared_ptr<mongo::CommandInvocation>)","s+":"B0"},{"a":"5633C3E8F996","b":"5633C03FE000","o":"3A91996","s":"_ZN5mongo12_GLOBAL__N_113InvokeCommand3runEv","C":"mongo::(anonymous namespace)::InvokeCommand::run()","s+":"236"},{"a":"5633C3E97311","b":"5633C03FE000","o":"3A99311","s":"_ZN5mongo12_GLOBAL__N_114RunCommandImpl11_runCommandEv","C":"mongo::(anonymous namespace)::RunCommandImpl::_runCommand()","s+":"2A1"},{"a":"5633C3E98DA6","b":"5633C03FE000","o":"3A9ADA6","s":"_ZN5mongo12_GLOBAL__N_114RunCommandImpl8_runImplEv","C":"mongo::(anonymous namespace)::RunCommandImpl::_runImpl()","s+":"96"},{"a":"5633C3E91736","b":"5633C03FE000","o":"3A93736","s":"_ZN5mongo12_GLOBAL__N_114RunCommandImpl3runEv","C":"mongo::(anonymous namespace)::RunCommandImpl::run()","s+":"136"},{"a":"5633C3E9A8F9","b":"5633C03FE000","o":"3A9C8F9","s":"_ZN5mongo12_GLOBAL__N_119ExecCommandDatabase12_commandExecEv","C":"mongo::(anonymous namespace)::ExecCommandDatabase::_commandExec()","s+":"1E9"},{"a":"5633C3EA0918","b":"5633C03FE000","o":"3AA2918","s":"_ZN5mongo19makeReadyFutureWithIZNOS_11future_util10AsyncStateINS_12_GLOBAL__N_119ExecCommandDatabaseEE13thenWithStateIZZNS3_14executeCommandESt10shared_ptrINS3_13HandleRequest16ExecutionContextEEENUlvE0_clEvEUlPT_E_EEDaOSC_EUlvE_EENS_6FutureINS_14future_details17UnwrappedTypeImplINSt13invoke_resultISF_JEE4typeEE4typeEEESF_","s+":"48"},{"a":"5633C3EA122C","b":"5633C03FE000","o":"3AA322C","s":"_ZZN5mongo15unique_functionIFvPNS_14future_details15SharedStateBaseEEE8makeImplIZNS1_10FutureImplINS1_8FakeVoidEE16makeContinuationIvZZNOS9_4thenINS_19CleanupFuturePolicyILb0EEEZNS_12_GLOBAL__N_114executeCommandESt10shared_ptrINSE_13HandleRequest16ExecutionContextEEEUlvE0_EEDaT_OT0_ENKUlvE1_clEvEUlPNS1_15SharedStateImplIS8_EESQ_E_EENS7_ISK_EESM_EUlS3_E_EEDaOSK_EN12SpecificImpl4callEOS3_","C":"mongo::unique_function<void (mongo::future_details::SharedStateBase*)>::makeImpl<mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::makeContinuation<void, mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::then<mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#2}>(mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#2}&&) &&::{lambda()#3}::operator()() const::{lambda(mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*, mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*)#1}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#2}&&)::{lambda(mongo::future_details::SharedStateBase*)#1}>(mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::makeContinuation<void, mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::then<mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#2}>(mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#2}&&) &&::{lambda()#3}::operator()() const::{lambda(mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*, mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*)#1}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#2}&&)::{lambda(mongo::future_details::SharedStateBase*)#1}&&)::SpecificImpl::call(mongo::future_details::SharedStateBase*&&)","s+":"1FC"},{"a":"5633C3E37B27","b":"5633C03FE000","o":"3A39B27","s":"_ZN5mongo14future_details15SharedStateBase20transitionToFinishedEv","C":"mongo::future_details::SharedStateBase::transitionToFinished()","s+":"107"},{"a":"5633C3EABD7C","b":"5633C03FE000","o":"3AADD7C","s":"_ZNO5mongo14future_details10FutureImplINS0_8FakeVoidEE17propagateResultToEPNS0_15SharedStateImplIS2_EE","C":"mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::propagateResultTo(mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*) &&","s+":"1AC"},{"a":"5633C3E999A3","b":"5633C03FE000","o":"3A9B9A3","s":"_ZZN5mongo15unique_functionIFvPNS_14future_details15SharedStateBaseEEE8makeImplIZNS1_10FutureImplINS1_8FakeVoidEE16makeContinuationIvZZNOS9_4thenINS_19CleanupFuturePolicyILb0EEEZNS_12_GLOBAL__N_114executeCommandESt10shared_ptrINSE_13HandleRequest16ExecutionContextEEEUlvE_EEDaT_OT0_ENKUlvE1_clEvEUlPNS1_15SharedStateImplIS8_EESQ_E_EENS7_ISK_EESM_EUlS3_E_EEDaOSK_EN12SpecificImpl4callEOS3_","C":"mongo::unique_function<void (mongo::future_details::SharedStateBase*)>::makeImpl<mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::makeContinuation<void, mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::then<mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#1}>(mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#1}&&) &&::{lambda()#3}::operator()() const::{lambda(mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*, mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*)#1}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#1}&&)::{lambda(mongo::future_details::SharedStateBase*)#1}>(mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::makeContinuation<void, mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::then<mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#1}>(mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#1}&&) &&::{lambda()#3}::operator()() const::{lambda(mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*, mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*)#1}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#1}&&)::{lambda(mongo::future_details::SharedStateBase*)#1}&&)::SpecificImpl::call(mongo::future_details::SharedStateBase*&&)","s+":"93"},{"a":"5633C3E37B27","b":"5633C03FE000","o":"3A39B27","s":"_ZN5mongo14future_details15SharedStateBase20transitionToFinishedEv","C":"mongo::future_details::SharedStateBase::transitionToFinished()","s+":"107"},{"a":"5633C3EA18D1","b":"5633C03FE000","o":"3AA38D1","s":"_ZN5mongo12_GLOBAL__N_114executeCommandESt10shared_ptrINS0_13HandleRequest16ExecutionContextEE","C":"mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)","s+":"641"},{"a":"5633C3EA238B","b":"5633C03FE000","o":"3AA438B","s":"_ZN5mongo12_GLOBAL__N_116receivedCommandsESt10shared_ptrINS0_13HandleRequest16ExecutionContextEE","C":"mongo::(anonymous namespace)::receivedCommands(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)","s+":"43B"},{"a":"5633C3EA2E78","b":"5633C03FE000","o":"3AA4E78","s":"_ZN5mongo12_GLOBAL__N_115CommandOpRunner3runEv","C":"mongo::(anonymous namespace)::CommandOpRunner::run()","s+":"48"},{"a":"5633C3E9580C","b":"5633C03FE000","o":"3A9780C","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageESt10unique_ptrIKNS0_5HooksESt14default_deleteIS8_EE","C":"mongo::ServiceEntryPointCommon::handleRequest(mongo::OperationContext*, mongo::Message const&, std::unique_ptr<mongo::ServiceEntryPointCommon::Hooks const, std::default_delete<mongo::ServiceEntryPointCommon::Hooks const> >)","s+":"37C"},{"a":"5633C3E878E0","b":"5633C03FE000","o":"3A898E0","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE","C":"mongo::ServiceEntryPointMongod::handleRequest(mongo::OperationContext*, mongo::Message const&)","s+":"50"},{"a":"5633C537E604","b":"5633C03FE000","o":"4F80604","s":"_ZN5mongo9transport15SessionWorkflow4Impl13_dispatchWorkEv","C":"mongo::transport::SessionWorkflow::Impl::_dispatchWork()","s+":"144"},{"a":"5633C537EE57","b":"5633C03FE000","o":"4F80E57","s":"_ZZNO5mongo14future_details10FutureImplISt10unique_ptrINS_9transport15SessionWorkflow4Impl8WorkItemESt14default_deleteIS6_EEE4thenINS_19CleanupFuturePolicyILb0EEEZNS5_15_doOneIterationEvEUlT_E_EEDaSE_OT0_ENKUlOS9_E_clESI_.isra.0","C":"mongo::future_details::FutureImpl<std::unique_ptr<mongo::transport::SessionWorkflow::Impl::WorkItem, std::default_delete<mongo::transport::SessionWorkflow::Impl::WorkItem> > >::then<mongo::CleanupFuturePolicy<false>, mongo::transport::SessionWorkflow::Impl::_doOneIteration()::{lambda(auto:1)#1}>(mongo::CleanupFuturePolicy<false>, mongo::transport::SessionWorkflow::Impl::_doOneIteration()::{lambda(auto:1)#1}&&) &&::{lambda(std::unique_ptr<mongo::transport::SessionWorkflow::Impl::WorkItem, std::default_delete<mongo::transport::SessionWorkflow::Impl::WorkItem> >&&)#1}::operator()(std::unique_ptr<mongo::transport::SessionWorkflow::Impl::WorkItem, std::default_delete<mongo::transport::SessionWorkflow::Impl::WorkItem> >&&) const [clone .isra.0]","s+":"47"},{"a":"5633C53806E5","b":"5633C03FE000","o":"4F826E5","s":"_ZN5mongo9transport15SessionWorkflow4Impl15_doOneIterationEv","C":"mongo::transport::SessionWorkflow::Impl::_doOneIteration()","s+":"535"},{"a":"5633C5380FCD","b":"5633C03FE000","o":"4F82FCD","s":"_ZZN5mongo15unique_functionIFvNS_6StatusEEE8makeImplIZNS_9transport15SessionWorkflow4Impl18_scheduleIterationEvEUlS1_E_EEDaOT_EN12SpecificImpl4callEOS1_","C":"mongo::unique_function<void (mongo::Status)>::makeImpl<mongo::transport::SessionWorkflow::Impl::_scheduleIteration()::{lambda(mongo::Status)#1}>(mongo::transport::SessionWorkflow::Impl::_scheduleIteration()::{lambda(mongo::Status)#1}&&)::SpecificImpl::call(mongo::Status&&)","s+":"5D"},{"a":"5633C5382E24","b":"5633C03FE000","o":"4F84E24","s":"_ZZN5mongo15unique_functionIFvNS_6StatusEEE8makeImplIZNS_9transport15SessionWorkflow4Impl15_captureContextES3_EUlS1_E_EEDaOT_EN12SpecificImpl4callEOS1_","C":"mongo::unique_function<void (mongo::Status)>::makeImpl<mongo::transport::SessionWorkflow::Impl::_captureContext(mongo::unique_function<void (mongo::Status)>)::{lambda(mongo::Status)#1}>(mongo::transport::SessionWorkflow::Impl::_captureContext(mongo::unique_function<void (mongo::Status)>)::{lambda(mongo::Status)#1}&&)::SpecificImpl::call(mongo::Status&&)","s+":"94"},{"a":"5633C74D5C24","b":"5633C03FE000","o":"70D7C24","s":"_ZZN5mongo15unique_functionIFvvEE8makeImplIZNS_9transport26ServiceExecutorSynchronous11SharedState8scheduleENS0_IFvNS_6StatusEEEEEUlvE0_EEDaOT_EN12SpecificImpl4callEv","C":"mongo::unique_function<void ()>::makeImpl<mongo::transport::ServiceExecutorSynchronous::SharedState::schedule(mongo::unique_function<void (mongo::Status)>)::{lambda()#2}>(mongo::transport::ServiceExecutorSynchronous::SharedState::schedule(mongo::unique_function<void (mongo::Status)>)::{lambda()#2}&&)::SpecificImpl::call()","s+":"C4"},{"a":"5633C74D8CDD","b":"5633C03FE000","o":"70DACDD","s":"_ZN5mongo9transport12_GLOBAL__N_17runFuncEPv","C":"mongo::transport::(anonymous namespace)::runFunc(void*)","s+":"3DD"},{"a":"7F2E09AFAB43","b":"7F2E09A66000","o":"94B43","s":"pthread_condattr_setpshared","s+":"513"},{"a":"7F2E09B8BBB4","b":"7F2E09A66000","o":"125BB4","s":"clone","s+":"44"}],"processInfo":{"mongodbVersion":"7.0.0","gitVersion":"37d84072b5c5b9fd723db5fa133fb202ad2317f1","compiledModules":[],"uname":{"sysname":"Linux","release":"5.15.0-82-generic","version":"#91-Ubuntu SMP Mon Aug 14 14:14:14 UTC 2023","machine":"x86_64"},"somap":[{"b":"5633C03FE000","elfType":3,"buildId":"36667979E9E7865B"},{"b":"7F2E09A66000","path":"/lib/x86_64-linux-gnu/libc.so.6","elfType":3,"buildId":"69389D485A9793DBE873F0EA2C93E02EFAA9AA3D"}]}}},"tags":[]}
{"t":{"$date":"2023-09-04T06:22:10.280+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"conn27","msg":"Frame","attr":{"frame":{"a":"5633C7E1AFB4","b":"5633C03FE000","o":"7A1CFB4","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_117getStackTraceImplERKNS1_7OptionsE.constprop.0","C":"mongo::stack_trace_detail::(anonymous namespace)::getStackTraceImpl(mongo::stack_trace_detail::(anonymous namespace)::Options const&) [clone .constprop.0]","s+":"224"}}}
#2

I was checking the logs on another crash and I found. This crash is happening every once in a while.

db.selfmade.ninja  | {"t":{"$date":"2023-09-04T07:54:55.867+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn214","msg":"client metadata","attr":{"remote":"172.18.0.1:38568","client":"conn214","doc":{"driver":{"name":"mongoc / ext-mongodb:PHP / PHPLIB ","version":"1.24.1 / 1.16.1 / 1.16.0 "},"os":{"type":"Linux","name":"Ubuntu","version":"22.04","architecture":"x86_64"},"platform":"PHP 8.1.2-1ubuntu2.13 cfg=0x03515620c9 posix=200809 stdc=201710 CC=GCC 11.3.0 CFLAGS=\"\" LDFLAGS=\"\""}}}
db.selfmade.ninja  | {"t":{"$date":"2023-09-04T07:54:55.868+00:00"},"s":"I",  "c":"NETWORK",  "id":6788700, "ctx":"conn214","msg":"Received first command on ingress connection since session start or auth handshake","attr":{"elapsedMillis":0}}

the context connection conn214 threw the error after over 20 mins from its first occurrence in log and then after 20 mins, it threw this followed by the stacktrace.

db.selfmade.ninja  | {"t":{"$date":"2023-09-04T08:13:20.370+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"conn214","msg":"WiredTiger error message","attr":{"error":22,"message":{"ts_sec":1693815200,"ts_usec":370599,"thread":"1:0x7f380caf0640","session_name":"WT_SESSION.get_value","category":"WT_VERB_DEFAULT","category_id":9,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"__wt_txn_context_prepare_check:19:not permitted in a prepared transaction","error_str":"Invalid argument","error_code":22}}}
db.selfmade.ninja  | {"t":{"$date":"2023-09-04T08:13:20.370+00:00"},"s":"F",  "c":"ASSERT",   "id":23083,   "ctx":"conn214","msg":"Invariant failure","attr":{"expr":"c->get_value(c, &value)","error":"BadValue: 22: Invalid argument","file":"src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp","line":1975}}
db.selfmade.ninja  | {"t":{"$date":"2023-09-04T08:13:20.370+00:00"},"s":"F",  "c":"ASSERT",   "id":23084,   "ctx":"conn214","msg":"\n\n***aborting after invariant() failure\n\n"}
db.selfmade.ninja  | {"t":{"$date":"2023-09-04T08:13:20.370+00:00"},"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"conn214","msg":"Writing fatal message","attr":{"message":"\n"}}
db.selfmade.ninja  | {"t":{"$date":"2023-09-04T08:13:20.370+00:00"},"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"conn214","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}
#3

@Satyam Can you help me on this? Thank you.

#4

I tried to repair the db, still the issue happens. Here is the repair log FYI

{"t":{"$date":"2023-09-04T13:36:44.119+00:00"},"s":"I",  "c":"NETWORK",  "id":4915701, "ctx":"main","msg":"Initialized wire specification","attr":{"spec":{"incomingExternalClient":{"minWireVersion":0,"maxWireVersion":21},"incomingInternalClient":{"minWireVersion":0,"maxWireVersion":21},"outgoing":{"minWireVersion":6,"maxWireVersion":21},"isInternalClient":true}}}
{"t":{"$date":"2023-09-04T13:36:44.120+00:00"},"s":"I",  "c":"CONTROL",  "id":23285,   "ctx":"main","msg":"Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'"}
{"t":{"$date":"2023-09-04T13:36:44.120+00:00"},"s":"I",  "c":"NETWORK",  "id":4648601, "ctx":"main","msg":"Implicit TCP FastOpen unavailable. If TCP FastOpen is required, set tcpFastOpenServer, tcpFastOpenClient, and tcpFastOpenQueueSize."}
{"t":{"$date":"2023-09-04T13:36:44.121+00:00"},"s":"I",  "c":"REPL",     "id":5123008, "ctx":"main","msg":"Successfully registered PrimaryOnlyService","attr":{"service":"TenantMigrationDonorService","namespace":"config.tenantMigrationDonors"}}
{"t":{"$date":"2023-09-04T13:36:44.121+00:00"},"s":"I",  "c":"REPL",     "id":5123008, "ctx":"main","msg":"Successfully registered PrimaryOnlyService","attr":{"service":"TenantMigrationRecipientService","namespace":"config.tenantMigrationRecipients"}}
{"t":{"$date":"2023-09-04T13:36:44.121+00:00"},"s":"I",  "c":"CONTROL",  "id":5945603, "ctx":"main","msg":"Multi threading initialized"}
{"t":{"$date":"2023-09-04T13:36:44.121+00:00"},"s":"I",  "c":"TENANT_M", "id":7091600, "ctx":"main","msg":"Starting TenantMigrationAccessBlockerRegistry"}
{"t":{"$date":"2023-09-04T13:36:44.121+00:00"},"s":"I",  "c":"CONTROL",  "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":1,"port":27017,"dbPath":"/data/db","architecture":"64-bit","host":"d89ca2c6d76b"}}
{"t":{"$date":"2023-09-04T13:36:44.121+00:00"},"s":"I",  "c":"CONTROL",  "id":23403,   "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"7.0.0","gitVersion":"37d84072b5c5b9fd723db5fa133fb202ad2317f1","openSSLVersion":"OpenSSL 3.0.2 15 Mar 2022","modules":[],"allocator":"tcmalloc","environment":{"distmod":"ubuntu2204","distarch":"x86_64","target_arch":"x86_64"}}}}
{"t":{"$date":"2023-09-04T13:36:44.121+00:00"},"s":"I",  "c":"CONTROL",  "id":51765,   "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"Ubuntu","version":"22.04"}}}
{"t":{"$date":"2023-09-04T13:36:44.121+00:00"},"s":"I",  "c":"CONTROL",  "id":21951,   "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"net":{"bindIp":"*"},"repair":true,"storage":{"dbPath":"/data/db"}}}}
{"t":{"$date":"2023-09-04T13:36:44.123+00:00"},"s":"I",  "c":"STORAGE",  "id":22270,   "ctx":"initandlisten","msg":"Storage engine to use detected by data files","attr":{"dbpath":"/data/db","storageEngine":"wiredTiger"}}
{"t":{"$date":"2023-09-04T13:36:44.123+00:00"},"s":"I",  "c":"STORAGE",  "id":22297,   "ctx":"initandlisten","msg":"Using the XFS filesystem is strongly recommended with the WiredTiger storage engine. See http://dochub.mongodb.org/core/prodnotes-filesystem","tags":["startupWarnings"]}
{"t":{"$date":"2023-09-04T13:36:44.123+00:00"},"s":"I",  "c":"STORAGE",  "id":22315,   "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=31637M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,remove=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=2000),statistics_log=(wait=0),json_output=(error,message),verbose=[recovery_progress:1,checkpoint_progress:1,compact_progress:1,backup:0,checkpoint:0,compact:0,evict:0,history_store:0,recovery:0,rts:0,salvage:0,tiered:0,timestamp:0,transaction:0,verify:0,log:0],"}}
{"t":{"$date":"2023-09-04T13:36:44.753+00:00"},"s":"I",  "c":"STORAGE",  "id":4795906, "ctx":"initandlisten","msg":"WiredTiger opened","attr":{"durationMillis":630}}
{"t":{"$date":"2023-09-04T13:36:44.753+00:00"},"s":"I",  "c":"RECOVERY", "id":23987,   "ctx":"initandlisten","msg":"WiredTiger recoveryTimestamp","attr":{"recoveryTimestamp":{"$timestamp":{"t":0,"i":0}}}}
{"t":{"$date":"2023-09-04T13:36:44.753+00:00"},"s":"I",  "c":"STORAGE",  "id":22316,   "ctx":"initandlisten","msg":"Repairing size cache"}
{"t":{"$date":"2023-09-04T13:36:44.753+00:00"},"s":"I",  "c":"STORAGE",  "id":22327,   "ctx":"initandlisten","msg":"Verify succeeded. Not salvaging.","attr":{"uri":"table:sizeStorer"}}
{"t":{"$date":"2023-09-04T13:36:44.753+00:00"},"s":"I",  "c":"STORAGE",  "id":22246,   "ctx":"initandlisten","msg":"Repairing catalog metadata"}
{"t":{"$date":"2023-09-04T13:36:44.754+00:00"},"s":"I",  "c":"STORAGE",  "id":22327,   "ctx":"initandlisten","msg":"Verify succeeded. Not salvaging.","attr":{"uri":"table:_mdb_catalog"}}
{"t":{"$date":"2023-09-04T13:36:44.756+00:00"},"s":"W",  "c":"CONTROL",  "id":22120,   "ctx":"initandlisten","msg":"Access control is not enabled for the database. Read and write access to data and configuration is unrestricted","tags":["startupWarnings"]}
{"t":{"$date":"2023-09-04T13:36:44.757+00:00"},"s":"W",  "c":"CONTROL",  "id":5123300, "ctx":"initandlisten","msg":"vm.max_map_count is too low","attr":{"currentValue":65530,"recommendedMinimum":1677720,"maxConns":838860},"tags":["startupWarnings"]}
{"t":{"$date":"2023-09-04T13:36:44.757+00:00"},"s":"I",  "c":"STORAGE",  "id":21027,   "ctx":"initandlisten","msg":"Repairing collection","attr":{"namespace":"admin.system.version"}}
{"t":{"$date":"2023-09-04T13:36:44.757+00:00"},"s":"I",  "c":"STORAGE",  "id":22327,   "ctx":"initandlisten","msg":"Verify succeeded. Not salvaging.","attr":{"uri":"table:collection-0-737012485405667727"}}
{"t":{"$date":"2023-09-04T13:36:44.758+00:00"},"s":"I",  "c":"INDEX",    "id":20295,   "ctx":"initandlisten","msg":"Validating internal structure","attr":{"index":"_id_","namespace":"admin.system.version"}}
{"t":{"$date":"2023-09-04T13:36:44.759+00:00"},"s":"I",  "c":"INDEX",    "id":20303,   "ctx":"initandlisten","msg":"validating collection","attr":{"namespace":"admin.system.version","uuid":{"uuid":{"$uuid":"f2df65c2-9924-4b78-af36-ecef5d4b9212"}}}}
{"t":{"$date":"2023-09-04T13:36:44.759+00:00"},"s":"I",  "c":"INDEX",    "id":20296,   "ctx":"initandlisten","msg":"Validating index consistency","attr":{"index":"_id_","namespace":"admin.system.version"}}
{"t":{"$date":"2023-09-04T13:36:44.759+00:00"},"s":"I",  "c":"INDEX",    "id":20306,   "ctx":"initandlisten","msg":"Validation complete for collection. No corruption found","attr":{"namespace":"admin.system.version","uuid":{"uuid":{"$uuid":"f2df65c2-9924-4b78-af36-ecef5d4b9212"}}}}
{"t":{"$date":"2023-09-04T13:36:44.759+00:00"},"s":"I",  "c":"STORAGE",  "id":21028,   "ctx":"initandlisten","msg":"Collection validation","attr":{"results":{"ns":"admin.system.version","uuid":{"$uuid":"f2df65c2-9924-4b78-af36-ecef5d4b9212"},"nInvalidDocuments":0,"nNonCompliantDocuments":0,"nrecords":1,"nIndexes":1,"keysPerIndex":{"_id_":1},"indexDetails":{"_id_":{"valid":true}}},"detailedResults":{"valid":true,"repaired":false,"warnings":[],"errors":[],"extraIndexEntries":[],"missingIndexEntries":[],"corruptRecords":[]}}}
{"t":{"$date":"2023-09-04T13:36:44.759+00:00"},"s":"I",  "c":"STORAGE",  "id":4934002, "ctx":"initandlisten","msg":"Validate did not make any repairs","attr":{"collection":"admin.system.version"}}
{"t":{"$date":"2023-09-04T13:36:44.760+00:00"},"s":"I",  "c":"NETWORK",  "id":4915702, "ctx":"initandlisten","msg":"Updated wire specification","attr":{"oldSpec":{"incomingExternalClient":{"minWireVersion":0,"maxWireVersion":21},"incomingInternalClient":{"minWireVersion":0,"maxWireVersion":21},"outgoing":{"minWireVersion":6,"maxWireVersion":21},"isInternalClient":true},"newSpec":{"incomingExternalClient":{"minWireVersion":0,"maxWireVersion":21},"incomingInternalClient":{"minWireVersion":17,"maxWireVersion":21},"outgoing":{"minWireVersion":17,"maxWireVersion":21},"isInternalClient":true}}}
{"t":{"$date":"2023-09-04T13:36:44.760+00:00"},"s":"I",  "c":"REPL",     "id":5853300, "ctx":"initandlisten","msg":"current featureCompatibilityVersion value","attr":{"featureCompatibilityVersion":"6.0","context":"startup"}}
{"t":{"$date":"2023-09-04T13:36:44.760+00:00"},"s":"I",  "c":"STORAGE",  "id":21029,   "ctx":"initandlisten","msg":"repairDatabase","attr":{"db":"admin"}}
{"t":{"$date":"2023-09-04T13:36:44.760+00:00"},"s":"I",  "c":"STORAGE",  "id":21027,   "ctx":"initandlisten","msg":"Repairing collection","attr":{"namespace":"admin.system.version"}}
{"t":{"$date":"2023-09-04T13:36:44.760+00:00"},"s":"I",  "c":"STORAGE",  "id":22327,   "ctx":"initandlisten","msg":"Verify succeeded. Not salvaging.","attr":{"uri":"table:collection-0-737012485405667727"}}
{"t":{"$date":"2023-09-04T13:36:44.760+00:00"},"s":"I",  "c":"INDEX",    "id":20295,   "ctx":"initandlisten","msg":"Validating internal structure","attr":{"index":"_id_","namespace":"admin.system.version"}}
{"t":{"$date":"2023-09-04T13:36:44.761+00:00"},"s":"I",  "c":"INDEX",    "id":20303,   "ctx":"initandlisten","msg":"validating collection","attr":{"namespace":"admin.system.version","uuid":{"uuid":{"$uuid":"f2df65c2-9924-4b78-af36-ecef5d4b9212"}}}}
{"t":{"$date":"2023-09-04T13:36:44.761+00:00"},"s":"I",  "c":"INDEX",    "id":20296,   "ctx":"initandlisten","msg":"Validating index consistency","attr":{"index":"_id_","namespace":"admin.system.version"}}
{"t":{"$date":"2023-09-04T13:36:44.761+00:00"},"s":"I",  "c":"INDEX",    "id":20306,   "ctx":"initandlisten","msg":"Validation complete for collection. No corruption found","attr":{"namespace":"admin.system.version","uuid":{"uuid":{"$uuid":"f2df65c2-9924-4b78-af36-ecef5d4b9212"}}}}
{"t":{"$date":"2023-09-04T13:36:44.761+00:00"},"s":"I",  "c":"STORAGE",  "id":21028,   "ctx":"initandlisten","msg":"Collection validation","attr":{"results":{"ns":"admin.system.version","uuid":{"$uuid":"f2df65c2-9924-4b78-af36-ecef5d4b9212"},"nInvalidDocuments":0,"nNonCompliantDocuments":0,"nrecords":1,"nIndexes":1,"keysPerIndex":{"_id_":1},"indexDetails":{"_id_":{"valid":true}}},"detailedResults":{"valid":true,"repaired":false,"warnings":[],"errors":[],"extraIndexEntries":[],"missingIndexEntries":[],"corruptRecords":[]}}}
{"t":{"$date":"2023-09-04T13:36:44.761+00:00"},"s":"I",  "c":"STORAGE",  "id":4934002, "ctx":"initandlisten","msg":"Validate did not make any repairs","attr":{"collection":"admin.system.version"}}
{"t":{"$date":"2023-09-04T13:36:44.762+00:00"},"s":"I",  "c":"CONTROL",  "id":6608200, "ctx":"initandlisten","msg":"Initializing cluster server parameters from disk"}
{"t":{"$date":"2023-09-04T13:36:44.762+00:00"},"s":"I",  "c":"CONTROL",  "id":20536,   "ctx":"initandlisten","msg":"Flow Control is enabled on this deployment"}
{"t":{"$date":"2023-09-04T13:36:44.762+00:00"},"s":"I",  "c":"FTDC",     "id":20625,   "ctx":"initandlisten","msg":"Initializing full-time diagnostic data capture","attr":{"dataDirectory":"/data/db/diagnostic.data"}}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"CONTROL",  "id":20537,   "ctx":"initandlisten","msg":"Finished checking dbs"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"REPL",     "id":4784900, "ctx":"initandlisten","msg":"Stepping down the ReplicationCoordinator for shutdown","attr":{"waitTimeMillis":15000}}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"REPL",     "id":4794602, "ctx":"initandlisten","msg":"Attempting to enter quiesce mode"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"-",        "id":6371601, "ctx":"initandlisten","msg":"Shutting down the FLE Crud thread pool"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"COMMAND",  "id":4784901, "ctx":"initandlisten","msg":"Shutting down the MirrorMaestro"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"SHARDING", "id":4784902, "ctx":"initandlisten","msg":"Shutting down the WaitForMajorityService"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"NETWORK",  "id":4784905, "ctx":"initandlisten","msg":"Shutting down the global connection pool"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"CONTROL",  "id":4784906, "ctx":"initandlisten","msg":"Shutting down the FlowControlTicketholder"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"-",        "id":20520,   "ctx":"initandlisten","msg":"Stopping further Flow Control ticket acquisitions."}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"CONTROL",  "id":4784908, "ctx":"initandlisten","msg":"Shutting down the PeriodicThreadToAbortExpiredTransactions"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"REPL",     "id":4784909, "ctx":"initandlisten","msg":"Shutting down the ReplicationCoordinator"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"SHARDING", "id":4784910, "ctx":"initandlisten","msg":"Shutting down the ShardingInitializationMongoD"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"REPL",     "id":4784911, "ctx":"initandlisten","msg":"Enqueuing the ReplicationStateTransitionLock for shutdown"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"-",        "id":4784912, "ctx":"initandlisten","msg":"Killing all operations for shutdown"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"-",        "id":4695300, "ctx":"initandlisten","msg":"Interrupted all currently running operations","attr":{"opsKilled":3}}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"TENANT_M", "id":5093807, "ctx":"initandlisten","msg":"Shutting down all TenantMigrationAccessBlockers on global shutdown"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"ASIO",     "id":22582,   "ctx":"TenantMigrationBlockerNet","msg":"Killing all outstanding egress activity."}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"ASIO",     "id":6529201, "ctx":"initandlisten","msg":"Network interface redundant shutdown","attr":{"state":"Stopped"}}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"ASIO",     "id":22582,   "ctx":"initandlisten","msg":"Killing all outstanding egress activity."}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"COMMAND",  "id":4784913, "ctx":"initandlisten","msg":"Shutting down all open transactions"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"REPL",     "id":4784914, "ctx":"initandlisten","msg":"Acquiring the ReplicationStateTransitionLock for shutdown"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"INDEX",    "id":4784915, "ctx":"initandlisten","msg":"Shutting down the IndexBuildsCoordinator"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"NETWORK",  "id":4784918, "ctx":"initandlisten","msg":"Shutting down the ReplicaSetMonitor"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"SHARDING", "id":4784921, "ctx":"initandlisten","msg":"Shutting down the MigrationUtilExecutor"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"ASIO",     "id":22582,   "ctx":"MigrationUtil-TaskExecutor","msg":"Killing all outstanding egress activity."}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"COMMAND",  "id":4784923, "ctx":"initandlisten","msg":"Shutting down the ServiceEntryPoint"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"CONTROL",  "id":4784925, "ctx":"initandlisten","msg":"Shutting down free monitoring"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"CONTROL",  "id":4784928, "ctx":"initandlisten","msg":"Shutting down the TTL monitor"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"CONTROL",  "id":6278511, "ctx":"initandlisten","msg":"Shutting down the Change Stream Expired Pre-images Remover"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"CONTROL",  "id":4784929, "ctx":"initandlisten","msg":"Acquiring the global lock for shutdown"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"CONTROL",  "id":4784930, "ctx":"initandlisten","msg":"Shutting down the storage engine"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"STORAGE",  "id":22320,   "ctx":"initandlisten","msg":"Shutting down journal flusher thread"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"STORAGE",  "id":22321,   "ctx":"initandlisten","msg":"Finished shutting down journal flusher thread"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"STORAGE",  "id":22322,   "ctx":"initandlisten","msg":"Shutting down checkpoint thread"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"STORAGE",  "id":22323,   "ctx":"initandlisten","msg":"Finished shutting down checkpoint thread"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"STORAGE",  "id":20282,   "ctx":"initandlisten","msg":"Deregistering all the collections"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"STORAGE",  "id":22317,   "ctx":"initandlisten","msg":"WiredTigerKVEngine shutting down"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"STORAGE",  "id":22318,   "ctx":"initandlisten","msg":"Shutting down session sweeper thread"}
{"t":{"$date":"2023-09-04T13:36:44.763+00:00"},"s":"I",  "c":"STORAGE",  "id":22319,   "ctx":"initandlisten","msg":"Finished shutting down session sweeper thread"}
{"t":{"$date":"2023-09-04T13:36:44.764+00:00"},"s":"I",  "c":"STORAGE",  "id":22324,   "ctx":"initandlisten","msg":"Closing WiredTiger in preparation for reconfiguring","attr":{"closeConfig":"leak_memory=true,use_timestamp=false,"}}
{"t":{"$date":"2023-09-04T13:36:44.772+00:00"},"s":"I",  "c":"STORAGE",  "id":4795905, "ctx":"initandlisten","msg":"WiredTiger closed","attr":{"durationMillis":8}}
{"t":{"$date":"2023-09-04T13:36:44.956+00:00"},"s":"I",  "c":"STORAGE",  "id":4795904, "ctx":"initandlisten","msg":"WiredTiger re-opened","attr":{"durationMillis":184}}
{"t":{"$date":"2023-09-04T13:36:44.956+00:00"},"s":"I",  "c":"STORAGE",  "id":22325,   "ctx":"initandlisten","msg":"Reconfiguring","attr":{"newConfig":"compatibility=(release=10.0)"}}
{"t":{"$date":"2023-09-04T13:36:44.959+00:00"},"s":"I",  "c":"STORAGE",  "id":4795903, "ctx":"initandlisten","msg":"Reconfigure complete","attr":{"durationMillis":3}}
{"t":{"$date":"2023-09-04T13:36:44.959+00:00"},"s":"I",  "c":"STORAGE",  "id":4795902, "ctx":"initandlisten","msg":"Closing WiredTiger","attr":{"closeConfig":"leak_memory=true,use_timestamp=false,"}}
{"t":{"$date":"2023-09-04T13:36:44.966+00:00"},"s":"I",  "c":"STORAGE",  "id":4795901, "ctx":"initandlisten","msg":"WiredTiger closed","attr":{"durationMillis":7}}
{"t":{"$date":"2023-09-04T13:36:44.966+00:00"},"s":"I",  "c":"STORAGE",  "id":22279,   "ctx":"initandlisten","msg":"shutdown: removing fs lock..."}
{"t":{"$date":"2023-09-04T13:36:44.966+00:00"},"s":"I",  "c":"-",        "id":4784931, "ctx":"initandlisten","msg":"Dropping the scope cache for shutdown"}
{"t":{"$date":"2023-09-04T13:36:44.966+00:00"},"s":"I",  "c":"FTDC",     "id":20626,   "ctx":"initandlisten","msg":"Shutting down full-time diagnostic data capture"}
{"t":{"$date":"2023-09-04T13:36:44.966+00:00"},"s":"I",  "c":"CONTROL",  "id":20565,   "ctx":"initandlisten","msg":"Now exiting"}
{"t":{"$date":"2023-09-04T13:36:44.966+00:00"},"s":"I",  "c":"CONTROL",  "id":23138,   "ctx":"initandlisten","msg":"Shutting down","attr":{"exitCode":0}}
#5

I dumped all data and moved to a different server, still the problem continues - same error happens and container restarts every now and then.

#6

I tried going back to mongodb 6.0, 6.0.9 also, still the error persists. The container fails with the same log every now and then.