Hi all! I’m experiencing random crashes in my MongoDB nodes due to a BSONObjectTooLarge error caused by the FTDC mechanism. After some research, I couldn’t find any bug reports or solutions related to this issue other than disabling the mechanism. However, that feels more like patching the symptoms rather than addressing the root cause. Has anyone else encountered this error (or bug) or knows a solution for it? Thanks!
MongoDB version: 7.0.12
Operating system: Ubuntu 22.04.5 LTS (Linux)
AWS EC2 flavor: m5.large
I’m attaching the last 40 log messages before the crash:
(click to expand)
{"t":{"$date":"2025-01-30T21:05:38.033-05:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn760148","msg":"Connection ended","attr":{"remote":"<REDACTED>","uuid":{"uuid":{"$uuid":"872c32db-0326-4244-a382-aa4df11b4317"}},"connectionId":760148,"connectionCount":9}}
{"t":{"$date":"2025-01-30T21:05:38.038-05:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"<REDACTED>","uuid":{"uuid":{"$uuid":"379c5409-c6e8-4ec8-9c13-127c2d698e39"}},"connectionId":760150,"connectionCount":10}}
{"t":{"$date":"2025-01-30T21:05:38.038-05:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn760150","msg":"client metadata","attr":{"remote":"<REDACTED>","client":"conn760150","negotiatedCompressors":[],"doc":{"driver":{"name":"mongo-java-driver|sync","version":"4.11.1"},"os":{"type":"Linux","name":"Linux","architecture":"aarch64","version":"5.10.230-223.885.amzn2.aarch64"},"platform":"Java/Amazon.com Inc./21.0.5+11-LTS"}}}
{"t":{"$date":"2025-01-30T21:05:38.038-05:00"},"s":"I", "c":"ACCESS", "id":6788604, "ctx":"conn760150","msg":"Auth metrics report","attr":{"metric":"acquireUser","micros":0}}
{"t":{"$date":"2025-01-30T21:05:38.042-05:00"},"s":"I", "c":"ACCESS", "id":5286306, "ctx":"conn760150","msg":"Successfully authenticated","attr":{"client":"<REDACTED>","isSpeculative":true,"isClusterMember":false,"mechanism":"SCRAM-SHA-256","user":"<REDACTED>","db":"<REDACTED>","result":0,"metrics":{"conversation_duration":{"micros":3954,"summary":{"0":{"step":1,"step_total":2,"duration_micros":60},"1":{"step":2,"step_total":2,"duration_micros":20}}}},"extraInfo":{}}}
{"t":{"$date":"2025-01-30T21:05:38.043-05:00"},"s":"I", "c":"NETWORK", "id":6788700, "ctx":"conn760150","msg":"Received first command on ingress connection since session start or auth handshake","attr":{"elapsedMillis":0}}
{"t":{"$date":"2025-01-30T21:05:39.695-05:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"<REDACTED>","uuid":{"uuid":{"$uuid":"6e385726-84f7-46f9-8bd3-6138b5fb9118"}},"connectionId":760151,"connectionCount":11}}
{"t":{"$date":"2025-01-30T21:05:39.695-05:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"<REDACTED>","uuid":{"uuid":{"$uuid":"09e5a006-5f45-4472-977e-22a2f2e0aa56"}},"connectionId":760152,"connectionCount":12}}
{"t":{"$date":"2025-01-30T21:05:39.696-05:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn760151","msg":"client metadata","attr":{"remote":"<REDACTED>","client":"conn760151","negotiatedCompressors":[],"doc":{"driver":{"name":"mongo-go-driver","version":"v1.10.3"},"os":{"type":"linux","architecture":"arm64"},"platform":"go1.17.13","application":{"name":"mongodb_exporter"}}}}
{"t":{"$date":"2025-01-30T21:05:39.696-05:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn760152","msg":"client metadata","attr":{"remote":"<REDACTED>","client":"conn760152","negotiatedCompressors":[],"doc":{"driver":{"name":"mongo-go-driver","version":"v1.10.3"},"os":{"type":"linux","architecture":"arm64"},"platform":"go1.17.13","application":{"name":"mongodb_exporter"}}}}
{"t":{"$date":"2025-01-30T21:05:39.696-05:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"<REDACTED>","uuid":{"uuid":{"$uuid":"47c42ba0-f1d5-4740-8fd4-b22de2d6926e"}},"connectionId":760153,"connectionCount":13}}
{"t":{"$date":"2025-01-30T21:05:39.697-05:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn760153","msg":"client metadata","attr":{"remote":"<REDACTED>","client":"conn760153","negotiatedCompressors":[],"doc":{"driver":{"name":"mongo-go-driver","version":"v1.10.3"},"os":{"type":"linux","architecture":"arm64"},"platform":"go1.17.13","application":{"name":"mongodb_exporter"}}}}
{"t":{"$date":"2025-01-30T21:05:39.697-05:00"},"s":"I", "c":"NETWORK", "id":6788700, "ctx":"conn760153","msg":"Received first command on ingress connection since session start or auth handshake","attr":{"elapsedMillis":0}}
{"t":{"$date":"2025-01-30T21:05:40.016-05:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"<REDACTED>","uuid":{"uuid":{"$uuid":"df549c6a-a63e-4b17-8f3f-3cc7323cca4b"}},"connectionId":760154,"connectionCount":14}}
{"t":{"$date":"2025-01-30T21:05:40.016-05:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn760154","msg":"client metadata","attr":{"remote":"<REDACTED>","client":"conn760154","negotiatedCompressors":[],"doc":{"driver":{"name":"mongo-java-driver|sync","version":"4.11.1"},"os":{"type":"Linux","name":"Linux","architecture":"aarch64","version":"5.10.230-223.885.amzn2.aarch64"},"platform":"Java/Amazon.com Inc./21.0.5+11-LTS"}}}
{"t":{"$date":"2025-01-30T21:05:40.017-05:00"},"s":"I", "c":"ACCESS", "id":6788604, "ctx":"conn760154","msg":"Auth metrics report","attr":{"metric":"acquireUser","micros":0}}
{"t":{"$date":"2025-01-30T21:05:40.020-05:00"},"s":"I", "c":"ACCESS", "id":5286306, "ctx":"conn760154","msg":"Successfully authenticated","attr":{"client":"<REDACTED>","isSpeculative":true,"isClusterMember":false,"mechanism":"SCRAM-SHA-256","user":"<REDACTED>","db":"<REDACTED>","result":0,"metrics":{"conversation_duration":{"micros":3969,"summary":{"0":{"step":1,"step_total":2,"duration_micros":53},"1":{"step":2,"step_total":2,"duration_micros":20}}}},"extraInfo":{}}}
{"t":{"$date":"2025-01-30T21:05:40.021-05:00"},"s":"I", "c":"NETWORK", "id":6788700, "ctx":"conn760154","msg":"Received first command on ingress connection since session start or auth handshake","attr":{"elapsedMillis":0}}
{"t":{"$date":"2025-01-30T21:05:40.426-05:00"},"s":"I", "c":"-", "id":20883, "ctx":"conn760152","msg":"Interrupted operation as its client disconnected","attr":{"opId":66692335}}
{"t":{"$date":"2025-01-30T21:05:40.427-05:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn760153","msg":"Connection ended","attr":{"remote":"<REDACTED>","uuid":{"uuid":{"$uuid":"47c42ba0-f1d5-4740-8fd4-b22de2d6926e"}},"connectionId":760153,"connectionCount":13}}
{"t":{"$date":"2025-01-30T21:05:40.427-05:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn760151","msg":"Connection ended","attr":{"remote":"<REDACTED>","uuid":{"uuid":{"$uuid":"6e385726-84f7-46f9-8bd3-6138b5fb9118"}},"connectionId":760151,"connectionCount":12}}
{"t":{"$date":"2025-01-30T21:05:40.427-05:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn760152","msg":"Connection ended","attr":{"remote":"<REDACTED>","uuid":{"uuid":{"$uuid":"09e5a006-5f45-4472-977e-22a2f2e0aa56"}},"connectionId":760152,"connectionCount":11}}
{"t":{"$date":"2025-01-30T21:05:42.369-05:00"},"s":"E", "c":"ASSERT", "id":23077, "ctx":"ftdc","msg":"Assertion","attr":{"error":"BSONObjectTooLarge: BSONObj size: 20079232 (0x1326280) is invalid. Size must be between 0 and 16793600(16MB) First element: start: new Date(1738289141000)","file":"src/mongo/bson/bsonobj.cpp","line":104}}
{"t":{"$date":"2025-01-30T21:05:42.370-05:00"},"s":"F", "c":"CONTROL", "id":6384300, "ctx":"ftdc","msg":"Writing fatal message","attr":{"message":"terminate() called. An exception is active; attempting to gather more information\n"}}
{"t":{"$date":"2025-01-30T21:05:42.370-05:00"},"s":"F", "c":"CONTROL", "id":6384300, "ctx":"ftdc","msg":"Writing fatal message","attr":{"message":"DBException::toString(): BSONObjectTooLarge: BSONObj size: 20079232 (0x1326280) is invalid. Size must be between 0 and 16793600(16MB) First element: start: new Date(1738289141000)\nActual exception type: mongo::error_details::ExceptionForImpl<(mongo::ErrorCodes::Error)10334, mongo::AssertionException>\n\n"}}
{"t":{"$date":"2025-01-30T21:05:42.370-05:00"},"s":"F", "c":"CONTROL", "id":6384300, "ctx":"ftdc","msg":"Writing fatal message","attr":{"message":"\n"}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31380, "ctx":"ftdc","msg":"BACKTRACE","attr":{"bt":{"backtrace":[{"a":"BD015B7953F4","b":"BD01546A0000","o":"70F53F4","s":"_ZN5mongo15printStackTraceEv","C":"mongo::printStackTrace()","s+":"44"},{"a":"BD015B79114C","b":"BD01546A0000","o":"70F114C","s":"_ZN5mongo12_GLOBAL__N_111myTerminateEv","C":"mongo::(anonymous namespace)::myTerminate()","s+":"11C"},{"a":"BD015B908A9C","b":"BD01546A0000","o":"7268A9C","s":"_ZN10__cxxabiv111__terminateEPFvvE","C":"__cxxabiv1::__terminate(void (*)())","s+":"C"},{"a":"BD015B908714","b":"BD01546A0000","o":"7268714","s":"__cxa_call_terminate","s+":"44"},{"a":"BD015B908184","b":"BD01546A0000","o":"7268184","s":"__gxx_personality_v0","s+":"84"},{"a":"E07A58B3E47C","b":"E07A58B30000","o":"E47C","s":"_Unwind_GetTextRelBase","s+":"1D8C"},{"a":"E07A58B3EA14","b":"E07A58B30000","o":"EA14","s":"_Unwind_Resume","s+":"80"},{"a":"BD01583B6580","b":"BD01546A0000","o":"3D16580","s":"_ZN5mongo23FTDCCollectorCollection7collectEPNS_6ClientE","C":"mongo::FTDCCollectorCollection::collect(mongo::Client*)","s+":"9C0"},{"a":"BD01583B966C","b":"BD01546A0000","o":"3D1966C","s":"_ZN5mongo14FTDCController6doLoopEv","C":"mongo::FTDCController::doLoop()","s+":"17C"},{"a":"BD01583B99CC","b":"BD01546A0000","o":"3D199CC","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_14FTDCController5startEvEUlvE0_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","C":"std::thread::_State_impl<std::thread::_Invoker<std::tuple<mongo::stdx::thread::thread<mongo::FTDCController::start()::{lambda()#2}, , 0>(mongo::FTDCController::start()::{lambda()#2})::{lambda()#1}> > >::_M_run()","s+":"7C"},{"a":"BD015B988CDC","b":"BD01546A0000","o":"72E8CDC","s":"execute_native_thread_routine","s+":"1C"},{"a":"E07A589FD5C8","b":"E07A58980000","o":"7D5C8","s":"pthread_condattr_setpshared","s+":"4F8"},{"a":"E07A58A65EDC","b":"E07A58980000","o":"E5EDC","s":"clone","s+":"5C"}],"processInfo":{"mongodbVersion":"7.0.12","gitVersion":"b6513ce0781db6818e24619e8a461eae90bc94fc","compiledModules":[],"uname":{"sysname":"Linux","release":"6.8.0-1017-aws","version":"#18~22.04.1-Ubuntu SMP Thu Oct 3 20:15:06 UTC 2024","machine":"aarch64"},"somap":[{"b":"BD01546A0000","elfType":3,"buildId":"F3A361AAA2BD766F"},{"b":"E07A58B30000","path":"/lib/aarch64-linux-gnu/libgcc_s.so.1","elfType":3,"buildId":"AD8704B8637D63A8F746C5E0CDEC870840569A81"},{"b":"E07A58980000","path":"/lib/aarch64-linux-gnu/libc.so.6","elfType":3,"buildId":"AA6E122FA39AE02D412AFB49D75E33281FCD2805"}]}}},"tags":[]}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"BD015B7953F4","b":"BD01546A0000","o":"70F53F4","s":"_ZN5mongo15printStackTraceEv","C":"mongo::printStackTrace()","s+":"44"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"BD015B79114C","b":"BD01546A0000","o":"70F114C","s":"_ZN5mongo12_GLOBAL__N_111myTerminateEv","C":"mongo::(anonymous namespace)::myTerminate()","s+":"11C"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"BD015B908A9C","b":"BD01546A0000","o":"7268A9C","s":"_ZN10__cxxabiv111__terminateEPFvvE","C":"__cxxabiv1::__terminate(void (*)())","s+":"C"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"BD015B908714","b":"BD01546A0000","o":"7268714","s":"__cxa_call_terminate","s+":"44"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"BD015B908184","b":"BD01546A0000","o":"7268184","s":"__gxx_personality_v0","s+":"84"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"E07A58B3E47C","b":"E07A58B30000","o":"E47C","s":"_Unwind_GetTextRelBase","s+":"1D8C"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"E07A58B3EA14","b":"E07A58B30000","o":"EA14","s":"_Unwind_Resume","s+":"80"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"BD01583B6580","b":"BD01546A0000","o":"3D16580","s":"_ZN5mongo23FTDCCollectorCollection7collectEPNS_6ClientE","C":"mongo::FTDCCollectorCollection::collect(mongo::Client*)","s+":"9C0"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"BD01583B966C","b":"BD01546A0000","o":"3D1966C","s":"_ZN5mongo14FTDCController6doLoopEv","C":"mongo::FTDCController::doLoop()","s+":"17C"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"BD01583B99CC","b":"BD01546A0000","o":"3D199CC","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_14FTDCController5startEvEUlvE0_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","C":"std::thread::_State_impl<std::thread::_Invoker<std::tuple<mongo::stdx::thread::thread<mongo::FTDCController::start()::{lambda()#2}, , 0>(mongo::FTDCController::start()::{lambda()#2})::{lambda()#1}> > >::_M_run()","s+":"7C"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"BD015B988CDC","b":"BD01546A0000","o":"72E8CDC","s":"execute_native_thread_routine","s+":"1C"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"E07A589FD5C8","b":"E07A58980000","o":"7D5C8","s":"pthread_condattr_setpshared","s+":"4F8"}}}
{"t":{"$date":"2025-01-30T21:05:42.521-05:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"ftdc","msg":"Frame","attr":{"frame":{"a":"E07A58A65EDC","b":"E07A58980000","o":"E5EDC","s":"clone","s+":"5C"}}}