Hi there!
I’m currently having an issue with the MongoDBClient in my NextJS application.
I’m running into a problem when inserting too much data from my app into the MongoDB on an AWS EBS PVC.
I’m inserting in batches by using the insertMany() function on the respective collection.
What happens is that when the persistent volume is about to be overfilled, the MongoDB pod is crashing and trying to restart. In the logs of the pod i see the following errors from WiredTiger:
{"t":{"$date":"2025-02-18T15:14:14.123+00:00"},"s":"E", "c":"WT", "id":22435, "ctx":"thread91","msg":"WiredTiger error message","attr":{"error":28,"message":{"ts_sec":1739891654,"ts_usec":123511,"thread":"1:0x7fe1183e8640","session_dhandle_name":"file:index-47-1825679840799448725.wt","session_name":"eviction-server","category":"WT_VERB_DEFAULT","category_id":9,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"__posix_file_write:621:/data/index-47-1825679840799448725.wt: handle-write: pwrite: failed to write 4096 bytes at offset 68657152","error_str":"No space left on device","error_code":28}}}
{"t":{"$date":"2025-02-18T15:14:14.123+00:00"},"s":"E", "c":"WT", "id":22435, "ctx":"thread91","msg":"WiredTiger error message","attr":{"error":28,"message":{"ts_sec":1739891654,"ts_usec":123627,"thread":"1:0x7fe1183e8640","session_name":"eviction-server","category":"WT_VERB_DEFAULT","category_id":9,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"__wt_evict_thread_run:344:cache eviction thread error","error_str":"No space left on device","error_code":28}}}
{"t":{"$date":"2025-02-18T15:14:14.123+00:00"},"s":"E", "c":"WT", "id":22435, "ctx":"thread91","msg":"WiredTiger error message","attr":{"error":-31804,"message":{"ts_sec":1739891654,"ts_usec":123649,"thread":"1:0x7fe1183e8640","session_name":"eviction-server","category":"WT_VERB_DEFAULT","category_id":9,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"__wt_evict_thread_run:344:the process must exit and restart","error_str":"WT_PANIC: WiredTiger library panic","error_code":-31804}}}
Now, from my MongoDBClient in the NextJs app, I receive only this error:
{"level":"debug","time":"2025-02-18T15:13:36.010Z","pid":1,"hostname":"avrotool-85b58d5946-7q7wq","err":{"type":"MongoBulkWriteError","message":"connect ECONNREFUSED 10.4.95.89:27017","stack":"MongoBulkWriteError: connect ECONNREFUSED 10.4.95.89:27017\n at executeCommands (/app/node_modules/mongodb/lib/bulk/common.js:343:23)\n at async BulkWriteShimOperation.execute (/app/node_modules/mongodb/lib/bulk/common.js:523:16)\n at async tryOperation (/app/node_modules/mongodb/lib/operations/execute_operation.js:203:20)\n at async executeOperation (/app/node_modules/mongodb/lib/operations/execute_operation.js:73:16)\n at async OrderedBulkOperation.execute (/app/node_modules/mongodb/lib/bulk/common.js:808:16)\n at async BulkWriteOperation.execute (/app/node_modules/mongodb/lib/operations/bulk_write.js:34:16)\n at async InsertManyOperation.execute (/app/node_modules/mongodb/lib/operations/insert.js:82:25)\n at async tryOperation (/app/node_modules/mongodb/lib/operations/execute_operation.js:203:20)\n at async executeOperation (/app/node_modules/mongodb/lib/operations/execute_operation.js:73:16)\n at async Collection.insertMany (/app/node_modules/mongodb/lib/collection.js:168:16)","errorResponse":{"type":"MongoServerSelectionError","message":"connect ECONNREFUSED 10.4.95.89:27017: connect ECONNREFUSED 10.4.95.89:27017: connect ECONNREFUSED 10.4.95.89:27017","stack":"MongoServerSelectionError: connect ECONNREFUSED 10.4.95.89:27017\n at Topology.selectServer (/app/node_modules/mongodb/lib/sdam/topology.js:320:38)\n at async tryOperation (/app/node_modules/mongodb/lib/operations/execute_operation.js:189:22)\n at async executeOperation (/app/node_modules/mongodb/lib/operations/execute_operation.js:73:16)\n at async executeCommands (/app/node_modules/mongodb/lib/bulk/common.js:327:22)\n at async BulkWriteShimOperation.execute (/app/node_modules/mongodb/lib/bulk/common.js:523:16)\n at async tryOperation (/app/node_modules/mongodb/lib/operations/execute_operation.js:203:20)\n at async executeOperation (/app/node_modules/mongodb/lib/operations/execute_operation.js:73:16)\n at async OrderedBulkOperation.execute (/app/node_modules/mongodb/lib/bulk/common.js:808:16)\n at async BulkWriteOperation.execute (/app/node_modules/mongodb/lib/operations/bulk_write.js:34:16)\n at async InsertManyOperation.execute (/app/node_modules/mongodb/lib/operations/insert.js:82:25)\ncaused by: MongoNetworkError: connect ECONNREFUSED 10.4.95.89:27017\n at Socket.<anonymous> (/app/node_modules/mongodb/lib/cmap/connect.js:285:44)\n at Object.onceWrapper (node:events:634:26)\n at Socket.emit (node:events:519:28)\n at emitErrorNT (node:internal/streams/destroy:169:8)\n at emitErrorCloseNT (node:internal/streams/destroy:128:3)\n at process.processTicksAndRejections (node:internal/process/task_queues:82:21)\ncaused by: Error: connect ECONNREFUSED 10.4.95.89:27017\n at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1607:16)\n at TCPConnectWrap.callbackTrampoline (node:internal/async_hooks:130:17)","reason":{"type":"ReplicaSetNoPrimary","servers":{"avrotool-mongodb-0.avrotool-mongodb-svc.temp-branch.svc.cluster.local:27017":{"address":"avrotool-mongodb-0.avrotool-mongodb-svc.temp-branch.svc.cluster.local:27017","type":"Unknown","hosts":[],"passives":[],"arbiters":[],"tags":{},"minWireVersion":0,"maxWireVersion":0,"roundTripTime":-1,"minRoundTripTime":0,"lastUpdateTime":1961459,"lastWriteDate":0,"error":{},"topologyVersion":null,"setName":null,"setVersion":null,"electionId":null,"logicalSessionTimeoutMinutes":null,"maxMessageSizeBytes":null,"maxWriteBatchSize":null,"maxBsonObjectSize":null,"primary":null,"me":null,"$clusterTime":null,"iscryptd":false}},"stale":false,"compatible":true,"heartbeatFrequencyMS":10000,"localThresholdMS":15,"setName":"avrotool-mongodb","maxElectionId":{"$oid":"7fffffff0000000000000001"},"maxSetVersion":1,"commonWireVersion":0,"logicalSessionTimeoutMinutes":null}},"reason":{"type":"ReplicaSetNoPrimary","servers":{"avrotool-mongodb-0.avrotool-mongodb-svc.temp-branch.svc.cluster.local:27017":{"address":"avrotool-mongodb-0.avrotool-mongodb-svc.temp-branch.svc.cluster.local:27017","type":"Unknown","hosts":[],"passives":[],"arbiters":[],"tags":{},"minWireVersion":0,"maxWireVersion":0,"roundTripTime":-1,"minRoundTripTime":0,"lastUpdateTime":1961459,"lastWriteDate":0,"error":{},"topologyVersion":null,"setName":null,"setVersion":null,"electionId":null,"logicalSessionTimeoutMinutes":null,"maxMessageSizeBytes":null,"maxWriteBatchSize":null,"maxBsonObjectSize":null,"primary":null,"me":null,"$clusterTime":null,"iscryptd":false}},"stale":false,"compatible":true,"heartbeatFrequencyMS":10000,"localThresholdMS":15,"setName":"avrotool-mongodb","maxElectionId":{"$oid":"7fffffff0000000000000001"},"maxSetVersion":1,"commonWireVersion":0,"logicalSessionTimeoutMinutes":null},"writeErrors":[],"result":{"insertedCount":277754, [...]
}
Mind that I have shortened the error message of the MongoDBClient because it is listing thousands if insert IDs.
What I don’t understand why is it that the MongoDB and/or WiredTiger doesn’t cancel the insert process earlier before it has to evict and the Pod gets restarted?
What I’m trying to achieve is that my NextJS Client get’s some kind of notification that import can not continue due to no free space left and then my NextJS Client would cancel the Import process.
Anything that I’m missing here?
Thanks a lot for you help!