Hello Everyone,
I get some error as below and due to this mongod instance was crashed.
2022-05-21T05:31:35.557+0000 E STORAGE [WTCheckpointThread] WiredTiger error (5) [1653111095:442272][11024:0x7f0c7bbe5700], file:index-35–2958543832295789442.wt, WT_SESSION.checkpoint: __posix_sync, 99: /var/lib/mongo/index-35–2958543832295789442.wt: handle-sync: fdatasync: Input/output error Raw: [1653111095:442272][11024:0x7f0c7bbe5700], file:index-35–2958543832295789442.wt, WT_SESSION.checkpoint: __posix_sync, 99: /var/lib/mongo/index-35–2958543832295789442.wt: handle-sync: fdatasync: Input/output error
2022-05-21T05:31:35.562+0000 E STORAGE [WTCheckpointThread] WiredTiger error (-31804) [1653111095:562169][11024:0x7f0c7bbe5700], file:index-35–2958543832295789442.wt, WT_SESSION.checkpoint: __wt_panic, 494: the process must exit and restart: WT_PANIC: WiredTiger library panic Raw: [1653111095:562169][11024:0x7f0c7bbe5700], file:index-35–2958543832295789442.wt, WT_SESSION.checkpoint: __wt_panic, 494: the process must exit and restart: WT_PANIC: WiredTiger library panic
2022-05-21T05:31:35.563+0000 F - [WTCheckpointThread] Fatal Assertion 50853 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 420
2022-05-21T05:31:35.563+0000 F - [WTCheckpointThread]
***aborting after fassert() failure
2022-05-21T05:31:35.605+0000 F - [WTCheckpointThread] Got signal: 6 (Aborted).
0x55e23ede6c21 0x55e23ede5e39 0x55e23ede631d 0x7f0c87524600 0x7f0c8717d3b7 0x7f0c8717eaa8 0x55e23d39f28b 0x55e23d4a2c76 0x55e23d50d741 0x55e23d325ed1 0x55e23d3262eb 0x55e23d4e0243 0x55e23d5de894 0x55e23d520559 0x55e23d52147b 0x55e23d5069ba 0x55e23d4800d6 0x55e23e831c31 0x55e23eef6630 0x7f0c8751ce75 0x7f0c872459bd
----- BEGIN BACKTRACE -----
{“backtrace”:[{“b”:“55E23C975000”,“o”:“2471C21”,“s”:"_ZN5mongo15printStackTraceERSo"},{“b”:“55E23C975000”,“o”:“2470E39”},{“b”:“55E23C975000”,“o”:“247131D”},{“b”:“7F0C87515000”,“o”:“F600”},{“b”:“7F0C87147000”,“o”:“363B7”,“s”:“gsignal”},{“b”:“7F0C87147000”,“o”:“37AA8”,“s”:“abort”},{“b”:“55E23C975000”,“o”:“A2A28B”,“s”:"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{“b”:“55E23C975000”,“o”:“B2DC76”},{“b”:“55E23C975000”,“o”:“B98741”},{“b”:“55E23C975000”,“o”:“9B0ED1”,“s”:"__wt_err_func"},{“b”:“55E23C975000”,“o”:“9B12EB”,“s”:"__wt_panic"},{“b”:“55E23C975000”,“o”:“B6B243”},{“b”:“55E23C975000”,“o”:“C69894”},{“b”:“55E23C975000”,“o”:“BAB559”},{“b”:“55E23C975000”,“o”:“BAC47B”,“s”:"__wt_txn_checkpoint"},{“b”:“55E23C975000”,“o”:“B919BA”},{“b”:“55E23C975000”,“o”:“B0B0D6”,“s”:"_ZN5mongo18WiredTigerKVEngine26WiredTigerCheckpointThread3runEv"},{“b”:“55E23C975000”,“o”:“1EBCC31”,“s”:"_ZN5mongo13BackgroundJob7jobBodyEv"},{“b”:“55E23C975000”,“o”:“2581630”},{“b”:“7F0C87515000”,“o”:“7E75”},{“b”:“7F0C87147000”,“o”:“FE9BD”,“s”:“clone”}],“processInfo”:{ “mongodbVersion” : “4.0.16”, “gitVersion” : “2a5433168a53044cb6b4fa8083e4cfd7ba142221”, “compiledModules” : [], “uname” : { “sysname” : “Linux”, “release” : “4.9.51-10.52.amzn1.x86_64”, “version” : “#1 SMP Fri Sep 29 01:16:19 UTC 2017”, “machine” : “x86_64” }, “somap” : [ { “b” : “55E23C975000”, “elfType” : 3, “buildId” : “5DD743B8BBEB16201177D6D09046399B64F5E029” }, { “b” : “7FFDC0775000”, “elfType” : 3, “buildId” : “E1FD0678C5C561D462A1DAA29BD08CC5361EF117” }, { “b” : “7F0C8893F000”, “path” : “/usr/lib64/libcurl.so.4”, “elfType” : 3, “buildId” : “9E7C58F4A4EE9752AF068429BA3DB14397639056” }, { “b” : “7F0C88725000”, “path” : “/lib64/libresolv.so.2”, “elfType” : 3, “buildId” : “AE1FDB1B0712ABF27B24A3F0E983619A30981750” }, { “b” : “7F0C882C6000”, “path” : “/lib64/libcrypto.so.10”, “elfType” : 3, “buildId” : “3270D2720328EEC2846C4B0D993582A0F657F54B” }, { “b” : “7F0C88055000”, “path” : “/lib64/libssl.so.10”, “elfType” : 3, “buildId” : “183215EA0DA6EE9C80A1E3A3319EC2905D1BF6E0” }, { “b” : “7F0C87E51000”, “path” : “/lib64/libdl.so.2”, “elfType” : 3, “buildId” : “D8859C267836C8AF28BBB238819141B6BF34F8D9” }, { “b” : “7F0C87C49000”, “path” : “/lib64/librt.so.1”, “elfType” : 3, “buildId” : “DB30FAB7C82FF7E06EE4913B0D3AB02C51DC0530” }, { “b” : “7F0C87947000”, “path” : “/lib64/libm.so.6”, “elfType” : 3, “buildId” : “19CC4E1B82AD44838E8DFD1FF893CA0CDAB7A5F5” }, { “b” : “7F0C87731000”, “path” : “/lib64/libgcc_s.so.1”, “elfType” : 3, “buildId” : “AC58019512A5359B077D2610DEA4AD6CF14CAC53” }, { “b” : “7F0C87515000”, “path” : “/lib64/libpthread.so.0”, “elfType” : 3, “buildId” : “E45DFDE9B88CABE002BE746AD486F620DE2B3E54” }, { “b” : “7F0C87147000”, “path” : “/lib64/libc.so.6”, “elfType” : 3, “buildId” : “75FE5BEDAD7802FF4A0268752CB4B4FFB293D1DC” }, { “b” : “7F0C88BC6000”, “path” : “/lib64/ld-linux-x86-64.so.2”, “elfType” : 3, “buildId” : “FD30DCC79F68A409A7C742A6943C16AE02E52986” }, { “b” : “7F0C86F21000”, “path” : “/usr/lib64/libnghttp2.so.14”, “elfType” : 3, “buildId” : “903C20D899C962C2E93B006E3BB7172C83D8ACF4” }, { “b” : “7F0C86D00000”, “path” : “/usr/lib64/libidn2.so.0”, “elfType” : 3, “buildId” : “5235BD50D3FB450683328735B730532020DEE4BF” }, { “b” : “7F0C86AD8000”, “path” : “/usr/lib64/libssh2.so.1”, “elfType” : 3, “buildId” : “E03CF776B39054AC3B2EA2AB15B161A858B5732C” }, { “b” : “7F0C86863000”, “path” : “/usr/lib64/libpsl.so.0”, “elfType” : 3, “buildId” : “09BFE69665CFEEC18F81D8C4A971DCA29310186C” }, { “b” : “7F0C86615000”, “path” : “/usr/lib64/libgssapi_krb5.so.2”, “elfType” : 3, “buildId” : “1BE9E6309ED365E35806E13FA9E23350D71F2513” }, { “b” : “7F0C8632E000”, “path” : “/usr/lib64/libkrb5.so.3”, “elfType” : 3, “buildId” : “9EE23694485D684651195C7B51766E47D0CB95E3” }, { “b” : “7F0C860FC000”, “path” : “/usr/lib64/libk5crypto.so.3”, “elfType” : 3, “buildId” : “FD5974E4861D56DFFFFC8BF5DB35E74B1C20ABD5” }, { “b” : “7F0C85EF9000”, “path” : “/usr/lib64/libcom_err.so.2”, “elfType” : 3, “buildId” : “5C01209C5AE1B1714F19B07EB58F2A1274B69DC8” }, { “b” : “7F0C85CA7000”, “path” : “/lib64/libldap-2.4.so.2”, “elfType” : 3, “buildId” : “97F36EE026428345EEB18EE7F9EFB048ADB415A7” }, { “b” : “7F0C85A98000”, “path” : “/lib64/liblber-2.4.so.2”, “elfType” : 3, “buildId” : “BBE520FB0B4F67D5F708D233C31E7047B759068D” }, { “b” : “7F0C85882000”, “path” : “/lib64/libz.so.1”, “elfType” : 3, “buildId” : “89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1” }, { “b” : “7F0C8556C000”, “path” : “/usr/lib64/libunistring.so.0”, “elfType” : 3, “buildId” : “2B090A6860553944846E3C227B6AD12F279B304F” }, { “b” : “7F0C851F6000”, “path” : “/usr/lib64/libicuuc.so.50”, “elfType” : 3, “buildId” : “06AB750458E6948B6F40F05E705996DB44ADDF9B” }, { “b” : “7F0C84FE7000”, “path” : “/usr/lib64/libkrb5support.so.0”, “elfType” : 3, “buildId” : “1B55330B231D45AF433F7D9DCA507C5FB0609780” }, { “b” : “7F0C84DE4000”, “path” : “/lib64/libkeyutils.so.1”, “elfType” : 3, “buildId” : “37A58210FA50C91E09387765408A92909468D25B” }, { “b” : “7F0C84BC9000”, “path” : “/usr/lib64/libsasl2.so.2”, “elfType” : 3, “buildId” : “354560FFC93703E5A80EEC8C66DF9E59DA335001” }, { “b” : “7F0C8496D000”, “path” : “/usr/lib64/libssl3.so”, “elfType” : 3, “buildId” : “D6B37F82A6D0A2DC428F305A2F5D9D78DB60D488” }, { “b” : “7F0C84746000”, “path” : “/usr/lib64/libsmime3.so”, “elfType” : 3, “buildId” : “2FF9EB779ACEAB03691997491A29D7275013D770” }, { “b” : “7F0C84419000”, “path” : “/usr/lib64/libnss3.so”, “elfType” : 3, “buildId” : “19D59FF9A54C790463BCD0349B0D78A4B8E1304E” }, { “b” : “7F0C841E9000”, “path” : “/usr/lib64/libnssutil3.so”, “elfType” : 3, “buildId” : “353A34D6411C8F45DB54E122B06E97EF6AEFD4F9” }, { “b” : “7F0C83FE5000”, “path” : “/lib64/libplds4.so”, “elfType” : 3, “buildId” : “D835EB19EC07E13AECBE3E80652846DAB04553CE” }, { “b” : “7F0C83DE0000”, “path” : “/lib64/libplc4.so”, “elfType” : 3, “buildId” : “668B5E4DEDFB2CABFEE75B097CC04CBBF8EC231D” }, { “b” : “7F0C83BA2000”, “path” : “/lib64/libnspr4.so”, “elfType” : 3, “buildId” : “8C8278A9557AA5D7942C94992BE85C2E73EA358B” }, { “b” : “7F0C825CF000”, “path” : “/usr/lib64/libicudata.so.50”, “elfType” : 3, “buildId” : “291EDB545286F945CDE3AF6F5CF24FA2F53FDDDA” }, { “b” : “7F0C8224C000”, “path” : “/usr/lib64/libstdc++.so.6”, “elfType” : 3, “buildId” : “699868CB2BF35D0936C954AF8AD53A001D6690EE” }, { “b” : “7F0C8202B000”, “path” : “/usr/lib64/libselinux.so.1”, “elfType” : 3, “buildId” : “F5054DC94443326819FBF3065CFDF5E4726F57EE” }, { “b” : “7F0C81DF4000”, “path” : “/lib64/libcrypt.so.1”, “elfType” : 3, “buildId” : “CA95D3723C3A72A75EAA6448328B06470DE7D1CC” }, { “b” : “7F0C81BF2000”, “path” : “/lib64/libfreebl3.so”, “elfType” : 3, “buildId” : “6C6DA5F0ECDD84E81C6A44036EBACD7AA77707EB” } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55e23ede6c21]
mongod(+0x2470E39) [0x55e23ede5e39]
mongod(+0x247131D) [0x55e23ede631d]
libpthread.so.0(+0xF600) [0x7f0c87524600]
libc.so.6(gsignal+0x37) [0x7f0c8717d3b7]
libc.so.6(abort+0x148) [0x7f0c8717eaa8]
mongod(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x55e23d39f28b]
mongod(+0xB2DC76) [0x55e23d4a2c76]
mongod(+0xB98741) [0x55e23d50d741]
mongod(__wt_err_func+0x90) [0x55e23d325ed1]
mongod(__wt_panic+0x39) [0x55e23d3262eb]
mongod(+0xB6B243) [0x55e23d4e0243]
mongod(+0xC69894) [0x55e23d5de894]
mongod(+0xBAB559) [0x55e23d520559]
mongod(__wt_txn_checkpoint+0x1DB) [0x55e23d52147b]
mongod(+0xB919BA) [0x55e23d5069ba]
mongod(_ZN5mongo18WiredTigerKVEngine26WiredTigerCheckpointThread3runEv+0x356) [0x55e23d4800d6]
mongod(_ZN5mongo13BackgroundJob7jobBodyEv+0x131) [0x55e23e831c31]
mongod(+0x2581630) [0x55e23eef6630]
libpthread.so.0(+0x7E75) [0x7f0c8751ce75]
libc.so.6(clone+0x6D) [0x7f0c872459bd]
----- END BACKTRACE -----
!