error":"Pipeline Completion Failed: MongoServerError: Shard shard1 not found\n at Connection.onMessage (/app/dist/server/node_modules/mongodb/lib/cmap/connection.js

Ops and Admin Sharding
1

Hi MongoDB team,
We caught an error of shard not found error for the keyId 7218453085146841111
but upon validating the keys, could see it’s still valid and not expired.

Shard not found error is below:

{"t":{"$date":"2023-06-08T14:23:29.769+00:00"},"s":"I", "c":"COMMAND", "id":51803, "ctx":"conn45454","msg":"Slow query","attr":{"type":"command","ns":"db_name.coll_name","appName":"app_name","command":{"update":"coll_name","updates":[{"q":{"_id":{"$oid":"order_id"}},"u":{"$set":{"status":"completion_error","updatedAt":{"$date":"2023-06-08T14:23:22.099Z"},"errorAt":{"$date":"2023-06-08T14:23:22.099Z"},"error":"Pipeline Completion Failed: MongoServerError: Shard shard1 not found\n at Connection.onMessage (/app/dist/server/node_modules/mongodb/lib/cmap/connection.js:230:30)\n at MessageStream.<anonymous> (/app/dist/server/node_modules/mongodb/lib/cm"}}}],"ordered":true,"lsid":{"id":{"$uuid":"51583070-b62c-4205-bccd-36433fb4321b"}},"txnNumber":42,"$clusterTime":{"clusterTime":{"$timestamp":{"t":1686234202,"i":88}},"signature":{"hash":{"$binary":{"base64":"foFdQlXf6yYGtYM6xzEqI9RQvMc=","subType":"0"}},"keyId":7218453085146841111}},"$db":"db_name"},"nShards":1,"nMatched":1,"nModified":1,"numYields":0,"reslen":185,"readConcern":{"level":"local","provenance":"implicitDefault"},"remote":"10.xx.xx.xx:58525","protocol":"op_msg","durationMillis":7604}}

Also, sometimes, it’s throwing an error of No keys found for HMAC that is valid for time while keyId 7218453085146841111 valid and not expired.

{"t":{"$date":"2023-06-17T06:18:17.876+00:00"},"s":"I", "c":"COMMAND", "id":51803, "ctx":"conn5041371","msg":"Slow query","attr":{"type":"command","ns":"db_name.coll_name","appName":"app_name","command":{"update":"coll_name","updates":[{"q":{"id":{"$oid":"order_id"}},"u":{"$set":{"status":"completion_error","updatedAt":{"$date":"2023-06-17T06:18:13.395Z"},"errorAt":{"$date":"2023-06-17T06:18:13.395Z"},"error":"Pipeline Completion Failed: MongoServerError: No keys found for HMAC that is valid for time: { ts: Timestamp(1686982689, 753) } with id: 7195944265355100184\n at Connection.onMessage (/app/dist/server/node_modules/mongodb/lib/cmap/connection.js:230:30)\n "}}}],"ordered":true,"lsid":{"id":{"$uuid":"e78f741a-4e75-4129-8b51-913e552684d9"}},"txnNumber":80,"$clusterTime":{"clusterTime":{"$timestamp":{"t":1686982692,"i":1185}},"signature":{"hash":{"$binary":{"base64":"AVGOSka5Tg6V4W48aUd/89tbBJs=","subType":"0"}},"keyId":7218453085146841111}},"$db":"db_name"},"nShards":1,"nMatched":1,"nModified":1,"numYields":0,"reslen":185,"readConcern":{"level":"local","provenance":"implicitDefault"},"remote":"10.xx.x.xxx:54362","protocol":"op_msg","durationMillis":1155}}

HMAC KeyId is below:

mongos> db.getSiblingDB("admin").system.keys.find().map(k => { return { _id: k._id, purpose: k.purpose, expiresAt: new Date(k.expiresAt.getTime()*1000) }})
[
	{
		"_id" : NumberLong("7218453085146841111"),
		"purpose" : "HMAC",
		"expiresAt" : ISODate("2023-07-04T06:44:38Z")
	},
	{
		"_id" : NumberLong("7218453085146841112"),
		"purpose" : "HMAC",
		"expiresAt" : ISODate("2023-10-02T06:44:38Z")
	},
	{
		"_id" : NumberLong("7259151692732564680"),
		"purpose" : "HMAC",
		"expiresAt" : ISODate("2023-12-31T06:44:38Z")
	}
]

Community MongoDB version used: 6.0.6-5

Please let me know, if required further more info.

Thanks in advance.

Best,
Mukesh