Getting "MongoServerSelectionError: Server selection timed out after 30000 ms" with AWS Lambda

[Shivkumar_Patel]

Hello,

I am getting the following error when I host my NodeJS Mongo code on AWS Lambda. I am using the node-mongodb-native library. I am using NodeJS with ExpressJS.

Below is my Mongo code:

const {MongoClient} = require('mongodb');

var client;

class MongoMethods {
    constructor(company, collection){
        this.company = company;
        this.collection = collection;
    }

    static async open() {
        if (client != undefined) return client
        client = await MongoClient.connect("mongodb+srv://xyz:abc@redacted.mongodb.net/abc", {useUnifiedTopology: false})
        return client
    }

    async get(accessParameters){
        try{
            await MongoMethods.open()
            const db = client.db(this.company);
            const collection = db.collection(this.collection);
            const val = await collection.findOne({"_id":accessParameters["at"]});
            return val
        }catch(err){
            console.log(err)
        }
    }

    async putIn(putParameters, type){
        try{
            await MongoMethods.open()
            const db = client.db(this.company);
            const collection = db.collection(this.collection);
            
            var existing = await collection.findOne({"_id":putParameters["at"]});

            if(type === "array"){
                var toAdd = existing !== undefined && putParameters["in"] in existing? existing[putParameters["in"]] : [];
                toAdd.push(putParameters["value"]);
            }else if(type === "dict"){
                var toAdd = existing !== undefined && putParameters["in"] in existing? existing[putParameters["in"]] : {};
                toAdd[putParameters["key"]] = putParameters["value"];
            }else{
                var toAdd = putParameters["value"];
            }
            
            await collection.updateOne({"_id":putParameters["at"]}, {"$set": {[putParameters["in"]]: toAdd}}, {upsert: true});
        }catch(err){
            console.log(err)
        }
    }
}

module.exports.MongoMethods = MongoMethods;

Here is an except of the ExpressJS code:

const serverless = require('serverless-http');
var express = require('express');
var cors = require('cors')
const bodyParser = require('body-parser');
const { validate, ValidationError } = require('express-superstruct');
const { MongoMethods } = require('./mongo.js')

MongoMethods.open()

const app = express()
app.use(cors())
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));


app.route('/journey/')
  .get(validate({uid : "string", site : "string"}), function (req, res) {
    var args = req.query
    var mongoClient = new MongoMethods(args.site, "journeys")
    mongoClient.get({at : args.uid}).catch(console.dir).then(result => {
      if(result == undefined){
        res.status(404).json({"Database Error" : "UID does not exists or site is not a valid website"})
      }
      res.status(200).json(result)
    })
  })
  
var SETTING = ["status", "config"]
app.route('/internal/')
  .get(validate({site : "string", setting : "string", page : "string"}), function (req, res) {
    var args = req.query;
    var toSearchSetting = args.setting

    if(!SETTING.includes(toSearchSetting)){
      res.status(404).json(toSearchSetting + " can not be found in our database")
    }


    var mongoClient = new MongoMethods("xyz", "config")
    mongoClient.get({at : args.site}).catch(console.dir).then(result => {
      if(result == undefined){
        res.status(404).json("The contents you are serching for can not be found in our database")
      }else if(toSearchSetting == "config"){
        result["abc"] = result["abc"][args.page]
        res.status(200).json(result)
      }else if(toSearchSetting == "status"){
        res.status(200).json(result["status"])
      }else{
        res.status(404).json("An unknown error has occured")
      }
    })
  })
  
module.exports.handler = serverless(app)

I have looked for solutions online. The two I have seen solutions:

1: Whitelist IP address. I cannot do this as the code runs on AWS Lambda, we do not control the IP address.

2: Set “useUnifiedTopology” to “false”. I did that and that did not work either.

Could someone help with this please?

[Vivek_Patel3 (Vivek Patel)]

Hey Shivkumar_Patel, Did you find a solution?

[Fedor_Vinogradov]

Same error here, any advice from anyone?

[Shivkumar_Patel]

@Fedor_Vinogradov @Vivek_Patel3 Yes, we found the solution. We modified our code and connection URL.

For our connection URL, we added the following: {useUnifiedTopology: true, useNewUrlParser: true, maxIdleTimeMS : 270000, minPoolSize : 2, maxPoolSize : 4} as the options parameters. It seems that adding maxIdleTimeMS, minPoolSize, and maxPoolSize fixed it. You can set the these options to your preferred value. The issue was that our connections were not closing and setting the maxIdleTimeMS allowed for the connection to be closed. Mongo docs online recommend the use of useUnifiedTopology and useNewUrlParser.

Additionally, our code also changed. We did not have 2 separate files anymore. We only have one application.js file. The code changes were minor. We moved the open() function out of the MongoMethods class and move it to the top of our application.js file and we called await open() everywhere we needed to connect to Mongo.

Here is an excerpt that show the new code and it’s use:

const serverless = require('serverless-http');
var express = require('express');
var cors = require('cors')
const bodyParser = require('body-parser');
const { validate, ValidationError } = require('express-superstruct');
const {MongoClient} = require('mongodb');
const e = require('express');

let client;

async function open() {
  if (client != undefined) return client
  
  client = await MongoClient.connect("mongodb+srv://abc:xyz@123.mongodb.net/?w=majority", {useUnifiedTopology: true, useNewUrlParser: true, maxIdleTimeMS : 270000, minPoolSize : 2, maxPoolSize : 4})
}


const app = express()
app.use(cors())
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));


app.route('/journey/')
  .get(validate({uid : "string", site : "string"}), function (req, res) {
    var args = req.query
    var mongoClient = new MongoMethods(args.site, "db")
    mongoClient.get({at : args.uid}).catch(console.dir).then(result => {
      if(result == undefined){
        res.status(404).json({"Database Error" : "UID does not exists or site is not a valid website"})
      }else{
        res.status(200).json(result["journey"])
      }
    })
  })


app.use((err, req, res, next) => {
  if(err instanceof ValidationError) {
      return res
          .status(400)
          .json({ message: err.message }); // Superstruct's verbose message
  }
});


class MongoMethods {
  constructor(company, collection){
      this.company = company;
      this.collection = collection;
  }

  async get(accessParameters){
      try{
          await open()
          const db = client.db(this.company);
          const collection = db.collection(this.collection);
          const val = await collection.findOne({"_id":accessParameters["at"]});
          return val
      }catch(err){
          console.log(err)
      }
  }
}

module.exports.handler = serverless(app)

After these changes, we have yet to see a timeout. Additionally, if it helps, you should enable API caching as well on AWS API Gateway (assuming this is what you use). Let me know if you have any questions about that.