Project Stack: MEAN Stack
Please help resolve this error.
Use Case:
An Investor should be search against a database(a collection for this project) and the search result should be stored in a collection.
Collection Involved:
- investors: List of people
- pmlaDbs: A database in which each investor should be searched. It has around 815K + documents. Total Collection size is approx. 140MB.
- pmlaChecks: Stores result of search performed for an Investor in pmlaDB.
The Node Process:
In req.body an array of Investor IDs are received.
For Each Investor ID, Investor document is retrieved, and the search is performed in pmlaDb.
Node method 1:
async function pmlaCheckMany(reqBody, user){
// this method triggers the below method and immediately sends response to Angular to inform the user that the search is in process
NowPmlaCheckMany(reqBody, refNo, user)
}
Node method 2: This is a very long method with reference to other methods within, hence not putting all the details below.
// This method processes the request
// Retrieves Investor doc for each investorId
// performs extensive search of various types in pmlaDb collection
// generates a new record for pmlaCheck collection for this investor and saves it
// it takes approx 5-7 seconds to complete each Investor search
// when approx 35 investorIds are searched there is no error.
// the error occurs when more than 35 investors are searched.
async function NowPmlaCheckMany(reqBody, refNo, user) {
const {investorIds, pmlaDbsTBC, checkType} = reqBody;
const userAuthData = await clientService.getMsClientIds(user, ['cob', 'pmla'], 'pmlaService.pmlaCheckMany');
const dbData = await PmlaDbMaster.find({isActive: true}).sort('ctrlName').lean().exec();
await checkIfPmlaDbsExistNEW(dbData, pmlaDbsTBC);
const fyData = await fyService.getFyMonthFmToday();
const fy = fyData.fy;
const mth = fyData.mth; // this is calendar mth
let batchId = await getPmlaBatchId();
const currentPmlaFiles = await getCurrentPmlaFiles(dbData, pmlaDbsTBC, user);
let fullPmlaDb;
let dbFilter = {};
let partDbCheck = false;
if(pmlaDbsTBC.checkAllDbs) {
} else {
partDbCheck = true;
let searchDBArr = []
for(const d of dbData) {
if(pmlaDbsTBC[d.ctrlName]) {
searchDBArr.push(d.ctrlName)
}
}
dbFilter = {ctrlName: { $in: searchDBArr}};
}
fullPmlaDb = await PmlaDb.find(dbFilter).lean().exec();
let pmlaCheckCount = 0;
let investorSrNos = '';
for(id of investorIds) {
const investor = await Investor.findById(id);
if(!investor) continue;
// more code........
///// THIS METHOD performs detailed search
let resData = await pmlaCheckForOneNew(fName, mName, lName, pan, country, isEntity, fullPmlaDb, partDbCheck, pmlaDbsTBC, dbData);
if(resData) {
// ... some more details added to resData obj
pmlaCheck = new PmlaCheckNew(resData);
pmlaCheck.isEntity = isEntity;
await pmlaCheck.save();
}
}
When this code is executed on my laptop, it runs fine. However, I get the below error when this is executed on a VPS. The VPS also has another webapp running using the same Mongo Service.
Error
Error: write EPIPE
at afterWriteDispatched (node:internal/stream_base_commons:160:15)
at writeGeneric (node:internal/stream_base_commons:151:3)
at Socket._writeGeneric (node:net:874:11)
at Socket._write (node:net:886:8)
at writeOrBuffer (node:internal/streams/writable:392:12)
at _write (node:internal/streams/writable:333:10)
at Writable.write (node:internal/streams/writable:337:10)
at Connection.write (/root/demo/node_modules/mongodb/lib/core/connection/connection.js:273:21)
at /root/demo/node_modules/mongodb/lib/core/connection/pool.js:1201:40
at /root/demo/node_modules/mongodb/lib/core/connection/pool.js:950:23
at process.processTicksAndRejections (node:internal/process/task_queues:77:11) {
name: 'MongoNetworkError'
}
MongoError: pool is draining, new operations prohibited
at Pool.write (/root/demo/node_modules/mongodb/lib/core/connection/pool.js:849:8)
at _command (/root/demo/node_modules/mongodb/lib/core/wireprotocol/command.js:149:10)
at command (/root/demo/node_modules/mongodb/lib/core/wireprotocol/command.js:28:5)
at Object.query (/root/demo/node_modules/mongodb/lib/core/wireprotocol/query.js:70:3)
at Server.query (/root/demo/node_modules/mongodb/lib/core/topologies/server.js:649:16)
at FindOperation.execute (/root/demo/node_modules/mongodb/lib/operations/find.js:38:12)
at /root/demo/node_modules/mongodb/lib/operations/execute_operation.js:144:17
at Server.selectServer (/root/demo/node_modules/mongodb/lib/core/topologies/server.js:837:3)
at Server.selectServer (/root/demo/node_modules/mongodb/lib/topologies/topology_base.js:342:32)
at executeWithServerSelection (/root/demo/node_modules/mongodb/lib/operations/execute_operation.js:131:12)
at /root/demo/node_modules/mongodb/lib/operations/execute_operation.js:70:9
at maybePromise (/root/demo/node_modules/mongodb/lib/utils.js:692:3)
at executeOperation (/root/demo/node_modules/mongodb/lib/operations/execute_operation.js:34:10)
at Cursor._initializeCursor (/root/demo/node_modules/mongodb/lib/core/cursor.js:541:7)
at Cursor._initializeCursor (/root/demo/node_modules/mongodb/lib/cursor.js:190:11)
at nextFunction (/root/demo/node_modules/mongodb/lib/core/cursor.js:744:10)
at Cursor._next (/root/demo/node_modules/mongodb/lib/core/cursor.js:209:5)
at /root/demo/node_modules/mongodb/lib/cursor.js:253:14
at maybePromise (/root/demo/node_modules/mongodb/lib/utils.js:692:3)
at Cursor.next (/root/demo/node_modules/mongodb/lib/cursor.js:238:12)
at FindOneOperation.execute (/root/demo/node_modules/mongodb/lib/operations/find_one.js:29:14)
at /root/demo/node_modules/mongodb/lib/operations/execute_operation.js:72:19
at maybePromise (/root/demo/node_modules/mongodb/lib/utils.js:692:3)
at executeOperation (/root/demo/node_modules/mongodb/lib/operations/execute_operation.js:34:10)
at Collection.<anonymous> (/root/demo/node_modules/mongodb/lib/collection.js:1114:12)
at Collection.deprecated [as findOne] (/root/demo/node_modules/mongodb/lib/utils.js:611:15)
at NativeCollection.<computed> [as findOne] (/root/demo/node_modules/mongoose/lib/drivers/node-mongodb-native/collection.js:244:33)
at NodeCollection.findOne (/root/demo/node_modules/mquery/lib/collection/node.js:42:19)