MongoDB Crash with "Invariant failure" : WT_NOTFOUND

Node: AWS R5
System: AMI 2
Kernal: 4.14.192-147.314.amzn2.x86_64
MongoDB Version: 4.4.0
Crash Log:

{"t":{"$date":"2021-10-13T21:03:21.298+08:00"},"s":"I",  "c":"ACCESS",   "id":20250,   "ctx":"conn4012336","msg":"Successful authentication","attr":{"mechanism":"SCRAM-SHA-256","principalName":"admin","authent
icationDatabase":"admin","client":"127.0.0.1:44028"}}
{"t":{"$date":"2021-10-13T21:03:21.298+08:00"},"s":"I",  "c":"ACCESS",   "id":20250,   "ctx":"conn4012337","msg":"Successful authentication","attr":{"mechanism":"SCRAM-SHA-256","principalName":"admin","authenticationDatabase":"admin","client":"127.0.0.1:44030"}}
{"t":{"$date":"2021-10-13T21:03:21.357+08:00"},"s":"I",  "c":"-",        "id":20883,   "ctx":"conn4012334","msg":"Interrupted operation as its client disconnected","attr":{"opId":1777138786}}
{"t":{"$date":"2021-10-13T21:03:21.357+08:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn4012333","msg":"connection ended","attr":{"remote":"127.0.0.1:44024","connectionCount":64}}
{"t":{"$date":"2021-10-13T21:03:21.357+08:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn4012334","msg":"connection ended","attr":{"remote":"127.0.0.1:44022","connectionCount":63}}
{"t":{"$date":"2021-10-13T21:03:21.357+08:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn4012337","msg":"connection ended","attr":{"remote":"127.0.0.1:44030","connectionCount":62}}
{"t":{"$date":"2021-10-13T21:03:21.357+08:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn4012335","msg":"connection ended","attr":{"remote":"127.0.0.1:44026","connectionCount":61}}
{"t":{"$date":"2021-10-13T21:03:21.357+08:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn4012336","msg":"connection ended","attr":{"remote":"127.0.0.1:44028","connectionCount":60}}
{"t":{"$date":"2021-10-13T21:03:21.570+08:00"},"s":"F",  "c":"-",        "id":23083,   "ctx":"conn3777212","msg":"Invariant failure","attr":{"expr":"ret","error":"UnknownError: -31803: WT_NOTFOUND: item not found","file":"src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp","line":1598}}
{"t":{"$date":"2021-10-13T21:03:21.570+08:00"},"s":"F",  "c":"-",        "id":23083,   "ctx":"conn3777216","msg":"Invariant failure","attr":{"expr":"ret","error":"UnknownError: -31803: WT_NOTFOUND: item not found","file":"src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp","line":1598}}
{"t":{"$date":"2021-10-13T21:03:21.571+08:00"},"s":"F",  "c":"-",        "id":23084,   "ctx":"conn3777212","msg":"\n\n***aborting after invariant() failure\n\n"}
{"t":{"$date":"2021-10-13T21:03:21.571+08:00"},"s":"F",  "c":"-",        "id":23084,   "ctx":"conn3777216","msg":"\n\n***aborting after invariant() failure\n\n"}
{"t":{"$date":"2021-10-13T21:03:21.573+08:00"},"s":"F",  "c":"CONTROL",  "id":4757800, "ctx":"conn3777216","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}
{"t":{"$date":"2021-10-13T21:03:21.814+08:00"},"s":"I",  "c":"CONTROL",  "id":31430,   "ctx":"conn3777216","msg":"Error collecting stack trace: {err}","attr":{"err":"unw_get_proc_name(7FF93AAD97E0): unspecified (general) error\nunw_get_proc_name(7FF93A750AE0): unspecified (general) error\nunw_get_proc_name(7FF93A751F88): unspecified (general) error\nunw_get_proc_name(7FF93AACF40B): unspecified (general) error\nunw_get_proc_name(7FF93A809E7F): unspecified (general) error\nunw_get_proc_name(7FF93AAD97E0): unspecified (general) error\nunw_get_proc_name(7FF93A750AE0): unspecified (general) error\nunw_get_proc_name(7FF93A751F88): unspecified (general) error\nunw_get_proc_name(7FF93AACF40B): unspecified (general) error\nunw_get_proc_name(7FF93A809E7F): unspecified (general) error\n"}}
{"t":{"$date":"2021-10-13T21:03:21.814+08:00"},"s":"F",  "c":"CONTROL",  "id":4757800, "ctx":"conn3777212","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}

{"t":{"$date":"2021-10-13T21:34:25.403+08:00"},"s":"W",  "c":"CONTROL",  "id":20698,   "ctx":"main","msg":"***** SERVER RESTARTED *****","tags":["startupWarnings"]}
{"t":{"$date":"2021-10-13T21:34:25.411+08:00"},"s":"I",  "c":"CONTROL",  "id":23285,   "ctx":"main","msg":"Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'"}
{"t":{"$date":"2021-10-13T21:34:25.544+08:00"},"s":"W",  "c":"ASIO",     "id":22601,   "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
{"t":{"$date":"2021-10-13T21:34:25.545+08:00"},"s":"I",  "c":"NETWORK",  "id":4648601, "ctx":"main","msg":"Implicit TCP FastOpen unavailable. If TCP FastOpen is required, set tcpFastOpenServer, tcpFastOpenClient, and tcpFastOpenQueueSize."}
{"t":{"$date":"2021-10-13T21:34:25.546+08:00"},"s":"I",  "c":"STORAGE",  "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":13144,"port":27017,"dbPath":"/var/lib/mongo","architecture":"64-bit","host":"aws_us_west1_r1"}}
{"t":{"$date":"2021-10-13T21:34:25.546+08:00"},"s":"I",  "c":"CONTROL",  "id":23403,   "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"4.4.0","gitVersion":"563487e100c4215e2dce98d0af2a6a5a2d67c5cf","openSSLVersion":"OpenSSL 1.0.2k-fips  26 Jan 2017","modules":[],"allocator":"tcmalloc","environment":{"distmod":"amazon2","distarch":"x86_64","target_arch":"x86_64"}}}}
{"t":{"$date":"2021-10-13T21:34:25.546+08:00"},"s":"I",  "c":"CONTROL",  "id":51765,   "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"Amazon Linux release 2 (Karoo)","version":"Kernel 4.14.192-147.314.amzn2.x86_64"}}}

Hi @Chi_Yao and welcome in the MongoDB Community :muscle: !

I found this ticket: https://jira.mongodb.org/browse/SERVER-50971

Looks like it could be related to a write-heavy collection with a TTL index but they never got enough information to reproduce.
Looks like it only happened in 4.4.0 and 4.4.1 so I would recommend to upgrade to 4.4.9.

Cheers,
Maxime.

1 Like

Hi, MaBeuLux88
This is what I am looking for,Thanks very much for your help!
I will upgrade MongoDB to 4.4.9 ,and monitor it for a long time!

Thanks and best regards
Chi Yao

1 Like

This topic was automatically closed 5 days after the last reply. New replies are no longer allowed.