Adjust system time cause Mongodb Detected unclean shutdown

Hi there,

I’m facing issues it seems adjst system time backward cause missing documents after restart mongodb.

Mongodb version: 4.4

Reproduce steps:

  1. set system backward a day, like “date -s “-1 day””
  2. insert some documents
  3. restart mongodb docker, then will find out all the documents inserted in Step2 are all gone…

Following is the log between shutting and restarting

{"t":{"$date":"2023-10-31T11:18:47.192+00:00"},"s":"I",  "c":"STORAGE",  "id":20320,   "ctx":"conn1","msg":"createCollection","attr":{"namespace":"test.user","uuidDisposition":"generated","uuid":{"uuid":{"$uuid":"27263bc2-a9c8-404d-9526-8cfc0b8c80fa"}},"options":{}}}
{"t":{"$date":"2023-10-31T11:18:47.203+00:00"},"s":"I",  "c":"INDEX",    "id":20345,   "ctx":"conn1","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"test.user","index":"_id_","commitTimestamp":{"$timestamp":{"t":1698751127,"i":1}}}}
{"t":{"$date":"2023-10-30T11:19:28.190+00:00"},"s":"I",  "c":"CONTROL",  "id":23377,   "ctx":"SignalHandler","msg":"Received signal","attr":{"signal":15,"error":"Terminated"}}
{"t":{"$date":"2023-10-30T11:19:28.190+00:00"},"s":"I",  "c":"CONTROL",  "id":23378,   "ctx":"SignalHandler","msg":"Signal was sent by kill(2)","attr":{"pid":0,"uid":0}}
{"t":{"$date":"2023-10-30T11:19:28.190+00:00"},"s":"I",  "c":"CONTROL",  "id":23381,   "ctx":"SignalHandler","msg":"will terminate after current cmd ends"}
{"t":{"$date":"2023-10-30T11:19:28.190+00:00"},"s":"I",  "c":"REPL",     "id":4784900, "ctx":"SignalHandler","msg":"Stepping down the ReplicationCoordinator for shutdown","attr":{"waitTimeMillis":10000}}
{"t":{"$date":"2023-10-30T11:19:28.190+00:00"},"s":"I",  "c":"COMMAND",  "id":4784901, "ctx":"SignalHandler","msg":"Shutting down the MirrorMaestro"}
{"t":{"$date":"2023-10-30T11:19:28.190+00:00"},"s":"I",  "c":"REPL",     "id":40441,   "ctx":"SignalHandler","msg":"Stopping TopologyVersionObserver"}
{"t":{"$date":"2023-10-30T11:19:28.190+00:00"},"s":"I",  "c":"REPL",     "id":40447,   "ctx":"TopologyVersionObserver","msg":"Stopped TopologyVersionObserver"}
{"t":{"$date":"2023-10-30T11:19:28.190+00:00"},"s":"I",  "c":"SHARDING", "id":4784902, "ctx":"SignalHandler","msg":"Shutting down the WaitForMajorityService"}
{"t":{"$date":"2023-10-30T11:19:28.191+00:00"},"s":"I",  "c":"CONTROL",  "id":4784903, "ctx":"SignalHandler","msg":"Shutting down the LogicalSessionCache"}
{"t":{"$date":"2023-10-30T11:19:28.191+00:00"},"s":"I",  "c":"NETWORK",  "id":20562,   "ctx":"SignalHandler","msg":"Shutdown: going to close listening sockets"}
{"t":{"$date":"2023-10-30T11:19:28.191+00:00"},"s":"I",  "c":"NETWORK",  "id":23017,   "ctx":"listener","msg":"removing socket file","attr":{"path":"/tmp/mongodb-27017.sock"}}
{"t":{"$date":"2023-10-30T11:19:28.191+00:00"},"s":"I",  "c":"NETWORK",  "id":4784905, "ctx":"SignalHandler","msg":"Shutting down the global connection pool"}
{"t":{"$date":"2023-10-30T11:19:28.191+00:00"},"s":"I",  "c":"STORAGE",  "id":4784906, "ctx":"SignalHandler","msg":"Shutting down the FlowControlTicketholder"}
{"t":{"$date":"2023-10-30T11:19:28.191+00:00"},"s":"I",  "c":"-",        "id":20520,   "ctx":"SignalHandler","msg":"Stopping further Flow Control ticket acquisitions."}
{"t":{"$date":"2023-10-30T11:19:28.191+00:00"},"s":"I",  "c":"REPL",     "id":4784907, "ctx":"SignalHandler","msg":"Shutting down the replica set node executor"}
{"t":{"$date":"2023-10-30T11:19:28.191+00:00"},"s":"I",  "c":"ASIO",     "id":22582,   "ctx":"ReplNodeDbWorkerNetwork","msg":"Killing all outstanding egress activity."}
{"t":{"$date":"2023-10-30T11:19:28.191+00:00"},"s":"I",  "c":"STORAGE",  "id":4784908, "ctx":"SignalHandler","msg":"Shutting down the PeriodicThreadToAbortExpiredTransactions"}
{"t":{"$date":"2023-10-30T11:19:28.192+00:00"},"s":"I",  "c":"STORAGE",  "id":4784934, "ctx":"SignalHandler","msg":"Shutting down the PeriodicThreadToDecreaseSnapshotHistoryCachePressure"}
{"t":{"$date":"2023-10-30T11:19:28.192+00:00"},"s":"I",  "c":"REPL",     "id":4784909, "ctx":"SignalHandler","msg":"Shutting down the ReplicationCoordinator"}
{"t":{"$date":"2023-10-30T11:19:28.192+00:00"},"s":"I",  "c":"REPL",     "id":21328,   "ctx":"SignalHandler","msg":"Shutting down replication subsystems"}
{"t":{"$date":"2023-10-30T11:19:28.192+00:00"},"s":"I",  "c":"REPL",     "id":21302,   "ctx":"SignalHandler","msg":"Stopping replication reporter thread"}
{"t":{"$date":"2023-10-30T11:19:28.192+00:00"},"s":"I",  "c":"REPL",     "id":21303,   "ctx":"SignalHandler","msg":"Stopping replication fetcher thread"}
{"t":{"$date":"2023-10-30T11:19:28.192+00:00"},"s":"I",  "c":"REPL",     "id":21304,   "ctx":"SignalHandler","msg":"Stopping replication applier thread"}
{"t":{"$date":"2023-10-30T11:19:28.192+00:00"},"s":"I",  "c":"REPL",     "id":21107,   "ctx":"BackgroundSync","msg":"Stopping replication producer"}
{"t":{"$date":"2023-10-30T11:19:38.698+00:00"},"s":"I",  "c":"CONTROL",  "id":23285,   "ctx":"main","msg":"Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'"}
{"t":{"$date":"2023-10-30T11:19:38.699+00:00"},"s":"I",  "c":"NETWORK",  "id":4648601, "ctx":"main","msg":"Implicit TCP FastOpen unavailable. If TCP FastOpen is required, set tcpFastOpenServer, tcpFastOpenClient, and tcpFastOpenQueueSize."}
{"t":{"$date":"2023-10-30T11:19:38.700+00:00"},"s":"I",  "c":"STORAGE",  "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":1,"port":27017,"dbPath":"/data/db","architecture":"64-bit","host":"2095516132c8"}}
{"t":{"$date":"2023-10-30T11:19:38.700+00:00"},"s":"I",  "c":"CONTROL",  "id":23403,   "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"4.4.25","gitVersion":"3e18c4c56048ddf22a6872edc111b542521ad1d5","openSSLVersion":"OpenSSL 1.1.1f  31 Mar 2020","modules":[],"allocator":"tcmalloc","environment":{"distmod":"ubuntu2004","distarch":"x86_64","target_arch":"x86_64"}}}}
{"t":{"$date":"2023-10-30T11:19:38.700+00:00"},"s":"I",  "c":"CONTROL",  "id":51765,   "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"Ubuntu","version":"20.04"}}}
{"t":{"$date":"2023-10-30T11:19:38.700+00:00"},"s":"I",  "c":"CONTROL",  "id":21951,   "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"net":{"bindIp":"*"},"replication":{"replSet":"RS"}}}}
{"t":{"$date":"2023-10-30T11:19:38.700+00:00"},"s":"W",  "c":"STORAGE",  "id":22271,   "ctx":"initandlisten","msg":"Detected unclean shutdown - Lock file is not empty","attr":{"lockFile":"/data/db/mongod.lock"}}
{"t":{"$date":"2023-10-30T11:19:38.700+00:00"},"s":"I",  "c":"STORAGE",  "id":22270,   "ctx":"initandlisten","msg":"Storage engine to use detected by data files","attr":{"dbpath":"/data/db","storageEngine":"wiredTiger"}}
{"t":{"$date":"2023-10-30T11:19:38.700+00:00"},"s":"W",  "c":"STORAGE",  "id":22302,   "ctx":"initandlisten","msg":"Recovering data from the last clean checkpoint."}

I adjust system from 2023-10-31 back to 2023-10-30 and insert some documents and then restart mongodb docker. It seems mongodb detect unclean shutdown and do rollback the inserted data at 2023-10-30.
As far as I know, mongodb shoud be able to handle time skew cases like leap seconds, mentioned in this topic: MongoDB and Leap Seconds | MongoDB Blog

Does anyone have any clues about it ?