Hi ,
Can anyone please help me to find out the reason based on only backtrace logs. Please note that this is not a recurring issue as of now and we noticed the same as first time.
Issue description : Mongodb crash issue
MongoDB version v4.0.2
OS details - CentOS release 6.10
Topology - 3 nodes a Replica set
There is nothing suspicious in messages and dmesg logs .
Logs :
2022-04-12T08:59:49.708-0500 F - [thread24725609] Invalid access at address: 0
2022-04-12T08:59:49.780-0500 F - [thread24725609] Got signal: 11 (Segmentation fault).
0x5645cd051fe1 0x5645cd0511f9 0x5645cd051866 0x7f93185757e0 0x5645cd13f0c3 0x5645cd13f19c 0x5645cd1e1a5a 0x5645cb81c616 0x5645cb81c696 0x5645cb81cce0 0x5645cb7b2cfb 0x5645cb7aaa8b 0x5645cb7aafd3 0x5645cb7ade17 0x5
645cb804999 0x7f931856daa1 0x7f93182bac4d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"5645CAC8F000","o":"23C2FE1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"5645CAC8F000","o":"23C21F9"},{"b":"5645CAC8F000","o":"23C2866"},{"b":"7F9318566000","o":"F7E0"},{"b":"5645CAC8F000","o":"2
4B00C3","s":"_ZN8tcmalloc11ThreadCache21ReleaseToCentralCacheEPNS0_8FreeListEmi"},{"b":"5645CAC8F000","o":"24B019C","s":"_ZN8tcmalloc11ThreadCache11ListTooLongEPNS0_8FreeListEm"},{"b":"5645CAC8F000","o":"2552A5A","
s":"_ZdlPvRKSt9nothrow_t"},{"b":"5645CAC8F000","o":"B8D616"},{"b":"5645CAC8F000","o":"B8D696"},{"b":"5645CAC8F000","o":"B8DCE0","s":"__wt_page_out"},{"b":"5645CAC8F000","o":"B23CFB","s":"__wt_evict"},{"b":"5645CAC8
F000","o":"B1BA8B"},{"b":"5645CAC8F000","o":"B1BFD3"},{"b":"5645CAC8F000","o":"B1EE17","s":"__wt_evict_thread_run"},{"b":"5645CAC8F000","o":"B75999"},{"b":"7F9318566000","o":"7AA1"},{"b":"7F93181D2000","o":"E8C4D",
"s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.2", "gitVersion" : "fc1573ba18aee42f97a3bb13b67af7d837826b47", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "2.6.32-754.23.1.el6.x86_64"
, "version" : "#1 SMP Thu Sep 26 12:05:41 UTC 2019", "machine" : "x86_64" }, "somap" : [ { "b" : "5645CAC8F000", "elfType" : 3, "buildId" : "86DF86886A5B030E7D7D4639DB20F4B807A3DAA4" }, { "b" : "7FFCE6E1B000", "elf
Type" : 3, "buildId" : "147632A13846D848909EB52353F6102CF932E1E5" }, { "b" : "7F9319894000", "path" : "/usr/lib64/libcurl.so.4", "elfType" : 3, "buildId" : "A38B9CE8AEAF277CBD8BC1298B1731E2C9A66192" }, { "b" : "7F9
31967A000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "4786A2A5D30B121601958E84D643C70C13C4FBA5" }, { "b" : "7F9319295000", "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "0435
315E0E6DC8BCF08DE1794B2A84D431DE835A" }, { "b" : "7F9319029000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "5A37D12297A649A37081AB63AC6B520444079986" }, { "b" : "7F9318E25000", "path" : "/lib64
/libdl.so.2", "elfType" : 3, "buildId" : "2AF795BFFD122309BA3359FEBABB5D0967403D17" }, { "b" : "7F9318C1D000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "552CEC3216281CCFD7FA6432C723D50163255823" },
{ "b" : "7F9318999000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "4AAEE970B045D8BF946578B9C7F3AB5CDE9AB44A" }, { "b" : "7F9318783000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "EDC
925E58FE28DCA536993EB13179C739F1E6566" }, { "b" : "7F9318566000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "4EA475CD3FD3B69B6C95D9381FA74B36DB4992EF" }, { "b" : "7F93181D2000", "path" : "/lib64
/libc.so.6", "elfType" : 3, "buildId" : "3E5ABB69E7969FB2C80A7D3637D62395D6C3F827" }, { "b" : "7F9319AE9000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "97AF4B77212F74CFF72B6C013E6AA2D74A97
EF60" }, { "b" : "7F9317FA0000", "path" : "/lib64/libidn.so.11", "elfType" : 3, "buildId" : "417A6ED67D904CD799091B56BDFADA0D594BB094" }, { "b" : "7F9317D4F000", "path" : "/lib64/libldap-2.4.so.2", "elfType" : 3, "
buildId" : "1E1F213BB990982FA49BCDD2CBBFB2DB680019AC" }, { "b" : "7F9317B0B000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "9A737F8BF10FC99C37CC404D3FC188F6E11FEDD9" }, { "b" : "7F9317824000
", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "8D3D6E28DF6EB3752642A7031AAC17D39EA4265D" }, { "b" : "7F93175F8000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "CC89B4C8CDCCD32BA6
10BC72784DC3B7E9BD9E19" }, { "b" : "7F93173F4000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "7EC54D6E88BB7D2C1284117C2A483496A01EAAF4" }, { "b" : "7F93171DE000", "path" : "/lib64/libz.so.1", "e
lfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "b" : "7F9316F8B000", "path" : "/usr/lib64/libssl3.so", "elfType" : 3, "buildId" : "9EA1D98414E89EC7EB3FD812F71A64C253F71B11" }, { "b" : "7F9
316D5E000", "path" : "/usr/lib64/libsmime3.so", "elfType" : 3, "buildId" : "05123600D28B4DF281811D84927C245F1CCFC011" }, { "b" : "7F9316A16000", "path" : "/usr/lib64/libnss3.so", "elfType" : 3, "buildId" : "7CAE725
EE3DCDC2FEB767F6B40639B2D7805CBD7" }, { "b" : "7F93167E7000", "path" : "/usr/lib64/libnssutil3.so", "elfType" : 3, "buildId" : "06AD65B988B6F761D5B334BF6AEE9CFED2D98AFF" }, { "b" : "7F93165E3000", "path" : "/lib64/
libplds4.so", "elfType" : 3, "buildId" : "CF853ECBE1D643A626D65701261A12ABE0828BF0" }, { "b" : "7F93163DE000", "path" : "/lib64/libplc4.so", "elfType" : 3, "buildId" : "810CDF602A65C7AF9CC7EA989BAFA4702B7738BF" },
{ "b" : "7F931619F000", "path" : "/lib64/libnspr4.so", "elfType" : 3, "buildId" : "36DBF3933E1ABF4DD774F7B456B5F406B632AA2A" }, { "b" : "7F9315F77000", "path" : "/usr/lib64/libssh2.so.1", "elfType" : 3, "buildId" :
"08D16725308A0FE306413488356B47AE8AAF71D3" }, { "b" : "7F9315D68000", "path" : "/lib64/liblber-2.4.so.2", "elfType" : 3, "buildId" : "B250057251FC715C5CCA27489506D7BBB1C9137C" }, { "b" : "7F9315B4E000", "path" : "
/usr/lib64/libsasl2.so.2", "elfType" : 3, "buildId" : "90F78E5E1AF40EE481268BEAF15F9A2E3E8EFBD2" }, { "b" : "7F9315943000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E0C522C589F775C324330BE
09CE67DC83950A213" }, { "b" : "7F9315740000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "b" : "7F9315509000", "path" : "/lib64/libcrypt.so.1", "e
lfType" : 3, "buildId" : "4D6D92ED643A94A2CABB7E3FF25FCEB29BE30269" }, { "b" : "7F93152EA000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "B4576BE308DDCF7BC31F7304E4734C3D846D0236" }, { "b" : "7F
93150E7000", "path" : "/lib64/libfreebl3.so", "elfType" : 3, "buildId" : "AFF1C795A3CF422C9F8AC32C7522F6376B1EA087" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x5645cd051fe1]
mongod(+0x23C21F9) [0x5645cd0511f9]
mongod(+0x23C2866) [0x5645cd051866]
libpthread.so.0(+0xF7E0) [0x7f93185757e0]
mongod(_ZN8tcmalloc11ThreadCache21ReleaseToCentralCacheEPNS0_8FreeListEmi+0xE3) [0x5645cd13f0c3]
mongod(_ZN8tcmalloc11ThreadCache11ListTooLongEPNS0_8FreeListEm+0x1C) [0x5645cd13f19c]
mongod(_ZdlPvRKSt9nothrow_t+0x26A) [0x5645cd1e1a5a]
mongod(+0xB8D616) [0x5645cb81c616]
mongod(+0xB8D696) [0x5645cb81c696]
mongod(__wt_page_out+0x590) [0x5645cb81cce0]
mongod(__wt_evict+0x19FB) [0x5645cb7b2cfb]
mongod(+0xB1BA8B) [0x5645cb7aaa8b]
mongod(+0xB1BFD3) [0x5645cb7aafd3]
mongod(__wt_evict_thread_run+0x77) [0x5645cb7ade17]
mongod(+0xB75999) [0x5645cb804999]
libpthread.so.0(+0x7AA1) [0x7f931856daa1]
libc.so.6(clone+0x6D) [0x7f93182bac4d]
----- END BACKTRACE -----