How to recover Mongodb from "Failed to refresh key cache"

Ops and Admin Replication
1

Mongo version: 5.0.5
OS: Debian GNU/Linux 10
RS config: PSA
Steps to reproduce:

  1. Setup a PSA cluster
  2. Brought down secondary data node and delete PVC.
  3. Bring up secondary node, it would not start.

Here is mongodb-0 logs:

"ctx":"monitoring-keys-for-HMAC","msg":"Failed to refresh key cache","attr":{"error":"ReadConcernMajorityNotAvailableYet: Read concern majority reads are currently not possible.","nextWakeupMillis":300000}

Here is mongodb-1 logs:

"ctx":"conn3","msg":"Failed to handle request","attr":{"error":"NotWritablePrimary: Not-primary error while processing 'find' operation  on 'config' database via fire-and-forget command execution."}

RS status:

"members" : [
    {
            "_id" : 0,
            "name" : "mongodb-0.mongodb-headless.mongodb-namespace.svc.cluster.local:27017",
            "health" : 1,
            "state" : 1,
            "stateStr" : "PRIMARY",
            "uptime" : 2965,
            "optime" : {
                    "ts" : Timestamp(1691657727, 1),
                    "t" : NumberLong(65)
            },
            "optimeDate" : ISODate("2023-08-10T08:55:27Z"),
            "lastAppliedWallTime" : ISODate("2023-08-10T08:55:27.277Z"),
            "lastDurableWallTime" : ISODate("2023-08-10T08:55:27.277Z"),
            "syncSourceHost" : "",
            "syncSourceId" : -1,
            "infoMessage" : "",
            "electionTime" : Timestamp(1691654877, 1),
            "electionDate" : ISODate("2023-08-10T08:07:57Z"),
            "configVersion" : 4,
            "configTerm" : 65,
            "self" : true,
            "lastHeartbeatMessage" : ""
    },
    {
            "_id" : 1,
            "name" : "mongodb-1.mongodb-headless.mongodb-namespace.svc.cluster.local:27017",
            "health" : 0,
            "state" : 8,
            "stateStr" : "(not reachable/healthy)",
            "uptime" : 0,
            "optime" : {
                    "ts" : Timestamp(0, 0),
                    "t" : NumberLong(-1)
            },
            "optimeDurable" : {
                    "ts" : Timestamp(0, 0),
                    "t" : NumberLong(-1)
            },
            "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
            "optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
            "lastAppliedWallTime" : ISODate("1970-01-01T00:00:00Z"),
            "lastDurableWallTime" : ISODate("1970-01-01T00:00:00Z"),
            "lastHeartbeat" : ISODate("2023-08-10T08:55:35.223Z"),
            "lastHeartbeatRecv" : ISODate("2023-08-10T08:49:02.915Z"),
            "pingMs" : NumberLong(0),
            "lastHeartbeatMessage" : "Error connecting to mongodb-1.mongodb-headless.mongodb-namespace.svc.cluster.local:27017 (10.128.6.137:27017) :: caused by :: Connection refused",
            "syncSourceHost" : "",
            "syncSourceId" : -1,
            "infoMessage" : "",
            "configVersion" : 4,
            "configTerm" : 65
    },
    {
            "_id" : 2,
            "name" : "mongodb-arbiter-0.mongodb-arbiter-headless.mongodb-namespace.svc.cluster.local:27017",
            "health" : 1,
            "state" : 7,
            "stateStr" : "ARBITER",
            "uptime" : 2869,
            "lastHeartbeat" : ISODate("2023-08-10T08:55:35.224Z"),
            "lastHeartbeatRecv" : ISODate("2023-08-10T08:55:35.232Z"),
            "pingMs" : NumberLong(0),
            "lastHeartbeatMessage" : "",
            "syncSourceHost" : "",
            "syncSourceId" : -1,
            "infoMessage" : "",
            "configVersion" : 4,
            "configTerm" : 65
    }
],
2

Did you solve the issue?
I’m getting the same error