Mongod crash with Got signal: 11 (Segmentation fault)

Hi ,

Can anyone please help me to find out the reason based on only backtrace logs. Please note that this is not a recurring issue as of now and we noticed the same as first time.

Issue description : Mongodb crash issue

MongoDB version v4.0.2
OS details - CentOS release 6.10
Topology - 3 nodes a Replica set
There is nothing suspicious in messages and dmesg logs .

Logs :

2022-04-12T08:59:49.708-0500 F -        [thread24725609] Invalid access at address: 0
2022-04-12T08:59:49.780-0500 F -        [thread24725609] Got signal: 11 (Segmentation fault).
 0x5645cd051fe1 0x5645cd0511f9 0x5645cd051866 0x7f93185757e0 0x5645cd13f0c3 0x5645cd13f19c 0x5645cd1e1a5a 0x5645cb81c616 0x5645cb81c696 0x5645cb81cce0 0x5645cb7b2cfb 0x5645cb7aaa8b 0x5645cb7aafd3 0x5645cb7ade17 0x5
645cb804999 0x7f931856daa1 0x7f93182bac4d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"5645CAC8F000","o":"23C2FE1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"5645CAC8F000","o":"23C21F9"},{"b":"5645CAC8F000","o":"23C2866"},{"b":"7F9318566000","o":"F7E0"},{"b":"5645CAC8F000","o":"2
4B00C3","s":"_ZN8tcmalloc11ThreadCache21ReleaseToCentralCacheEPNS0_8FreeListEmi"},{"b":"5645CAC8F000","o":"24B019C","s":"_ZN8tcmalloc11ThreadCache11ListTooLongEPNS0_8FreeListEm"},{"b":"5645CAC8F000","o":"2552A5A","
s":"_ZdlPvRKSt9nothrow_t"},{"b":"5645CAC8F000","o":"B8D616"},{"b":"5645CAC8F000","o":"B8D696"},{"b":"5645CAC8F000","o":"B8DCE0","s":"__wt_page_out"},{"b":"5645CAC8F000","o":"B23CFB","s":"__wt_evict"},{"b":"5645CAC8
F000","o":"B1BA8B"},{"b":"5645CAC8F000","o":"B1BFD3"},{"b":"5645CAC8F000","o":"B1EE17","s":"__wt_evict_thread_run"},{"b":"5645CAC8F000","o":"B75999"},{"b":"7F9318566000","o":"7AA1"},{"b":"7F93181D2000","o":"E8C4D",
"s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.2", "gitVersion" : "fc1573ba18aee42f97a3bb13b67af7d837826b47", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "2.6.32-754.23.1.el6.x86_64"
, "version" : "#1 SMP Thu Sep 26 12:05:41 UTC 2019", "machine" : "x86_64" }, "somap" : [ { "b" : "5645CAC8F000", "elfType" : 3, "buildId" : "86DF86886A5B030E7D7D4639DB20F4B807A3DAA4" }, { "b" : "7FFCE6E1B000", "elf
Type" : 3, "buildId" : "147632A13846D848909EB52353F6102CF932E1E5" }, { "b" : "7F9319894000", "path" : "/usr/lib64/libcurl.so.4", "elfType" : 3, "buildId" : "A38B9CE8AEAF277CBD8BC1298B1731E2C9A66192" }, { "b" : "7F9
31967A000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "4786A2A5D30B121601958E84D643C70C13C4FBA5" }, { "b" : "7F9319295000", "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "0435
315E0E6DC8BCF08DE1794B2A84D431DE835A" }, { "b" : "7F9319029000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "5A37D12297A649A37081AB63AC6B520444079986" }, { "b" : "7F9318E25000", "path" : "/lib64
/libdl.so.2", "elfType" : 3, "buildId" : "2AF795BFFD122309BA3359FEBABB5D0967403D17" }, { "b" : "7F9318C1D000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "552CEC3216281CCFD7FA6432C723D50163255823" },
{ "b" : "7F9318999000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "4AAEE970B045D8BF946578B9C7F3AB5CDE9AB44A" }, { "b" : "7F9318783000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "EDC
925E58FE28DCA536993EB13179C739F1E6566" }, { "b" : "7F9318566000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "4EA475CD3FD3B69B6C95D9381FA74B36DB4992EF" }, { "b" : "7F93181D2000", "path" : "/lib64
/libc.so.6", "elfType" : 3, "buildId" : "3E5ABB69E7969FB2C80A7D3637D62395D6C3F827" }, { "b" : "7F9319AE9000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "97AF4B77212F74CFF72B6C013E6AA2D74A97
EF60" }, { "b" : "7F9317FA0000", "path" : "/lib64/libidn.so.11", "elfType" : 3, "buildId" : "417A6ED67D904CD799091B56BDFADA0D594BB094" }, { "b" : "7F9317D4F000", "path" : "/lib64/libldap-2.4.so.2", "elfType" : 3, "
buildId" : "1E1F213BB990982FA49BCDD2CBBFB2DB680019AC" }, { "b" : "7F9317B0B000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "9A737F8BF10FC99C37CC404D3FC188F6E11FEDD9" }, { "b" : "7F9317824000
", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "8D3D6E28DF6EB3752642A7031AAC17D39EA4265D" }, { "b" : "7F93175F8000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "CC89B4C8CDCCD32BA6
10BC72784DC3B7E9BD9E19" }, { "b" : "7F93173F4000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "7EC54D6E88BB7D2C1284117C2A483496A01EAAF4" }, { "b" : "7F93171DE000", "path" : "/lib64/libz.so.1", "e
lfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "b" : "7F9316F8B000", "path" : "/usr/lib64/libssl3.so", "elfType" : 3, "buildId" : "9EA1D98414E89EC7EB3FD812F71A64C253F71B11" }, { "b" : "7F9
316D5E000", "path" : "/usr/lib64/libsmime3.so", "elfType" : 3, "buildId" : "05123600D28B4DF281811D84927C245F1CCFC011" }, { "b" : "7F9316A16000", "path" : "/usr/lib64/libnss3.so", "elfType" : 3, "buildId" : "7CAE725
EE3DCDC2FEB767F6B40639B2D7805CBD7" }, { "b" : "7F93167E7000", "path" : "/usr/lib64/libnssutil3.so", "elfType" : 3, "buildId" : "06AD65B988B6F761D5B334BF6AEE9CFED2D98AFF" }, { "b" : "7F93165E3000", "path" : "/lib64/
libplds4.so", "elfType" : 3, "buildId" : "CF853ECBE1D643A626D65701261A12ABE0828BF0" }, { "b" : "7F93163DE000", "path" : "/lib64/libplc4.so", "elfType" : 3, "buildId" : "810CDF602A65C7AF9CC7EA989BAFA4702B7738BF" },
{ "b" : "7F931619F000", "path" : "/lib64/libnspr4.so", "elfType" : 3, "buildId" : "36DBF3933E1ABF4DD774F7B456B5F406B632AA2A" }, { "b" : "7F9315F77000", "path" : "/usr/lib64/libssh2.so.1", "elfType" : 3, "buildId" :
 "08D16725308A0FE306413488356B47AE8AAF71D3" }, { "b" : "7F9315D68000", "path" : "/lib64/liblber-2.4.so.2", "elfType" : 3, "buildId" : "B250057251FC715C5CCA27489506D7BBB1C9137C" }, { "b" : "7F9315B4E000", "path" : "
/usr/lib64/libsasl2.so.2", "elfType" : 3, "buildId" : "90F78E5E1AF40EE481268BEAF15F9A2E3E8EFBD2" }, { "b" : "7F9315943000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E0C522C589F775C324330BE
09CE67DC83950A213" }, { "b" : "7F9315740000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "b" : "7F9315509000", "path" : "/lib64/libcrypt.so.1", "e
lfType" : 3, "buildId" : "4D6D92ED643A94A2CABB7E3FF25FCEB29BE30269" }, { "b" : "7F93152EA000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "B4576BE308DDCF7BC31F7304E4734C3D846D0236" }, { "b" : "7F
93150E7000", "path" : "/lib64/libfreebl3.so", "elfType" : 3, "buildId" : "AFF1C795A3CF422C9F8AC32C7522F6376B1EA087" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x5645cd051fe1]
 mongod(+0x23C21F9) [0x5645cd0511f9]
 mongod(+0x23C2866) [0x5645cd051866]
 libpthread.so.0(+0xF7E0) [0x7f93185757e0]
 mongod(_ZN8tcmalloc11ThreadCache21ReleaseToCentralCacheEPNS0_8FreeListEmi+0xE3) [0x5645cd13f0c3]
 mongod(_ZN8tcmalloc11ThreadCache11ListTooLongEPNS0_8FreeListEm+0x1C) [0x5645cd13f19c]
 mongod(_ZdlPvRKSt9nothrow_t+0x26A) [0x5645cd1e1a5a]
 mongod(+0xB8D616) [0x5645cb81c616]
 mongod(+0xB8D696) [0x5645cb81c696]
 mongod(__wt_page_out+0x590) [0x5645cb81cce0]
 mongod(__wt_evict+0x19FB) [0x5645cb7b2cfb]
 mongod(+0xB1BA8B) [0x5645cb7aaa8b]
 mongod(+0xB1BFD3) [0x5645cb7aafd3]
 mongod(__wt_evict_thread_run+0x77) [0x5645cb7ade17]
 mongod(+0xB75999) [0x5645cb804999]
 libpthread.so.0(+0x7AA1) [0x7f931856daa1]
 libc.so.6(clone+0x6D) [0x7f93182bac4d]
-----  END BACKTRACE  -----

Hi @Vinay_Jaiswal,

I have absolutely no idea without more context, logs, etc.

But I can suggest to upgrade to MongoDB 5.0 which might solve the issue as the team is constantly improving the software.

Also MongoDB 4.0 will be deprecated at the end of April 2022 (check our legacy support policy), so there is very little chance to get some investigations done on this old version.

If you choose to update, you’ll have to follow this route:

Cheers,
Maxime.