I have a replication with 1 primary, 1 member and 1 arbiter.
The primary was down this morning. When checking log, I saw the primary cannot connect to slave and it down till we restart the server
{"t":{"$date":"2021-01-15T03:30:03.814+07:00"},"s":"I", "c":"REPL", "id":21216, "ctx":"ReplCoord-92921","msg":"Member is now in state RS_DOWN","attr":{"hostAndPort":"SLAVE.com:27017","heartbeatMessage":"no response within election timeout period"}}
{"t":{"$date":"2021-01-15T03:30:04.026+07:00"},"s":"I", "c":"CONNPOOL", "id":22566, "ctx":"ReplNetwork","msg":"Ending connection due to bad connection status","attr":{"hostAndPort":"SLAVE.com:27017","error":"CallbackCanceled: Callback was canceled","numOpenConns":0}}
{"t":{"$date":"2021-01-15T03:30:04.028+07:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"ReplNetwork","msg":"Connecting","attr":{"hostAndPort":"SLAVE.com:27017"}}
{"t":{"$date":"2021-01-15T03:30:05.333+07:00"},"s":"I", "c":"CONNPOOL", "id":22566, "ctx":"ReplNetwork","msg":"Ending connection due to bad connection status","attr":{"hostAndPort":"img.SLAVE.com:27017","error":"CallbackCanceled: Callback was canceled","numOpenConns":0}}
{"t":{"$date":"2021-01-15T03:30:05.333+07:00"},"s":"I", "c":"REPL", "id":21216, "ctx":"ReplCoord-92921","msg":"Member is now in state RS_DOWN","attr":{"hostAndPort":"img.SLAVE.com:27017","heartbeatMessage":"Request 8595498 timed out, deadline was 2021-01-15T03:30:05.333+07:00, op was RemoteCommand 8595498 -- target:[img.SLAVE.com:27017] db:admin expDate:2021-01-15T03:30:05.333+07:00 cmd:{ replSetHeartbeat: \"ssg\", configVersion: 2, configTerm: 2, hbv: 1, from: \"207.148.116.98:27017\", fromId: 0, term: 2 }"}}
{"t":{"$date":"2021-01-15T03:30:05.333+07:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"ReplNetwork","msg":"Connecting","attr":{"hostAndPort":"img.SLAVE.com:27017"}}
{"t":{"$date":"2021-01-15T03:30:05.805+07:00"},"s":"I", "c":"REPL", "id":21809, "ctx":"ReplCoord-92920","msg":"Can't see a majority of the set, relinquishing primary"}
{"t":{"$date":"2021-01-15T03:30:05.805+07:00"},"s":"I", "c":"REPL", "id":21475, "ctx":"ReplCoord-92920","msg":"Stepping down from primary in response to heartbeat"}
{"t":{"$date":"2021-01-15T03:30:05.807+07:00"},"s":"I", "c":"REPL", "id":21343, "ctx":"RstlKillOpThread","msg":"Starting to kill user operations"}
{"t":{"$date":"2021-01-15T03:30:05.807+07:00"},"s":"I", "c":"REPL", "id":21344, "ctx":"RstlKillOpThread","msg":"Stopped killing user operations"}
{"t":{"$date":"2021-01-15T03:30:05.807+07:00"},"s":"I", "c":"REPL", "id":21340, "ctx":"RstlKillOpThread","msg":"State transition ops metrics","attr":{"metrics":{"lastStateTransition":"stepDown","userOpsKilled":0,"userOpsRunning":2}}}
{"t":{"$date":"2021-01-15T03:30:05.809+07:00"},"s":"I", "c":"REPL", "id":21358, "ctx":"ReplCoord-92920","msg":"Replica set state transition","attr":{"newState":"SECONDARY","oldState":"PRIMARY"}}
{"t":{"$date":"2021-01-15T03:30:05.814+07:00"},"s":"F", "c":"-", "id":23079, "ctx":"monitoring-keys-for-HMAC","msg":"Invariant failure","attr":{"expr":"opCtx != nullptr && _opCtx == nullptr","file":"src/mongo/db/client.cpp","line":126}}
{"t":{"$date":"2021-01-15T03:30:05.814+07:00"},"s":"F", "c":"-", "id":23080, "ctx":"monitoring-keys-for-HMAC","msg":"\n\n***aborting after invariant() failure\n\n"}
{"t":{"$date":"2021-01-15T03:30:05.815+07:00"},"s":"F", "c":"CONTROL", "id":4757800, "ctx":"monitoring-keys-for-HMAC","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}
{"t":{"$date":"2021-01-15T03:30:05.822+07:00"},"s":"F", "c":"-", "id":23079, "ctx":"TopologyVersionObserver","msg":"Invariant failure","attr":{"expr":"opCtx != nullptr && _opCtx == nullptr","file":"src/mongo/db/client.cpp","line":126}}
{"t":{"$date":"2021-01-15T03:30:05.822+07:00"},"s":"F", "c":"-", "id":23080, "ctx":"TopologyVersionObserver","msg":"\n\n***aborting after invariant() failure\n\n"}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31431, "ctx":"monitoring-keys-for-HMAC","msg":"BACKTRACE: {bt}","attr":{"bt":{"backtrace":[{"a":"563B70884D11","b":"563B6DBBF000","o":"2CC5D11","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1E1"},{"a":"563B70886349","b":"563B6DBBF000","o":"2CC7349","s":"_ZN5mongo15printStackTraceEv","s+":"29"},{"a":"563B70883BA6","b":"563B6DBBF000","o":"2CC4BA6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"},{"a":"7F88D472A630","b":"7F88D471B000","o":"F630","s":"_L_unlock_13","s+":"34"},{"a":"7F88D4383387","b":"7F88D434D000","o":"36387","s":"gsignal","s+":"37"},{"a":"7F88D4384A78","b":"7F88D434D000","o":"37A78","s":"abort","s+":"148"},{"a":"563B6EAC6119","b":"563B6DBBF000","o":"F07119","s":"_ZN5mongo15invariantFailedEPKcS1_j","s+":"12C"},{"a":"563B6EA9B630","b":"563B6DBBF000","o":"EDC630","s":"_ZN5mongo6Client19setOperationContextEPNS_16OperationContextE.cold.135","s+":"18"},{"a":"563B707533E9","b":"563B6DBBF000","o":"2B943E9","s":"_ZN5mongo14ServiceContext20makeOperationContextEPNS_6ClientE","s+":"129"},{"a":"563B70747E97","b":"563B6DBBF000","o":"2B88E97","s":"_ZN5mongo6Client20makeOperationContextEv","s+":"27"},{"a":"563B70546F62","b":"563B6DBBF000","o":"2987F62","s":"_ZN5mongo21KeysCollectionManager14PeriodicRunner18_doPeriodicRefreshEPNS_14ServiceContextENSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEENS_8DurationISt5ratioILl1ELl1000EEEE","s+":"162"},{"a":"563B70548EC3","b":"563B6DBBF000","o":"2989EC3","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_21KeysCollectionManager14PeriodicRunner5startEPNS3_14ServiceContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEENS3_8DurationISt5ratioILl1ELl1000EEEEEUlvE_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"93"},{"a":"563B70A30E7F","b":"563B6DBBF000","o":"2E71E7F","s":"execute_native_thread_routine","s+":"F"},{"a":"7F88D4722EA5","b":"7F88D471B000","o":"7EA5","s":"start_thread","s+":"C5"},{"a":"7F88D444B8DD","b":"7F88D434D000","o":"FE8DD","s":"clone","s+":"6D"}],"processInfo":{"mongodbVersion":"4.4.1","gitVersion":"ad91a93a5a31e175f5cbf8c69561e788bbc55ce1","compiledModules":[],"uname":{"sysname":"Linux","release":"3.10.0-957.27.2.el7.x86_64","version":"#1 SMP Mon Jul 29 17:46:05 UTC 2019","machine":"x86_64"},"somap":[{"b":"563B6DBBF000","elfType":3,"buildId":"6AF036CAFAF898729FE132E06C67BAC328F17731"},{"b":"7F88D471B000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"2B482B3BAE79DEF4E5BC9791BC6BBDAE0E93E359"},{"b":"7F88D434D000","path":"/lib64/libc.so.6","elfType":3,"buildId":"D78066A9C36F5FD63E2F6AC851AE3515C4C9792A"}]}}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"563B70884D11","b":"563B6DBBF000","o":"2CC5D11","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1E1"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"563B70886349","b":"563B6DBBF000","o":"2CC7349","s":"_ZN5mongo15printStackTraceEv","s+":"29"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"563B70883BA6","b":"563B6DBBF000","o":"2CC4BA6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"7F88D472A630","b":"7F88D471B000","o":"F630","s":"_L_unlock_13","s+":"34"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"7F88D4383387","b":"7F88D434D000","o":"36387","s":"gsignal","s+":"37"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"7F88D4384A78","b":"7F88D434D000","o":"37A78","s":"abort","s+":"148"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"563B6EAC6119","b":"563B6DBBF000","o":"F07119","s":"_ZN5mongo15invariantFailedEPKcS1_j","s+":"12C"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"563B6EA9B630","b":"563B6DBBF000","o":"EDC630","s":"_ZN5mongo6Client19setOperationContextEPNS_16OperationContextE.cold.135","s+":"18"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"563B707533E9","b":"563B6DBBF000","o":"2B943E9","s":"_ZN5mongo14ServiceContext20makeOperationContextEPNS_6ClientE","s+":"129"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"563B70747E97","b":"563B6DBBF000","o":"2B88E97","s":"_ZN5mongo6Client20makeOperationContextEv","s+":"27"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"563B70546F62","b":"563B6DBBF000","o":"2987F62","s":"_ZN5mongo21KeysCollectionManager14PeriodicRunner18_doPeriodicRefreshEPNS_14ServiceContextENSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEENS_8DurationISt5ratioILl1ELl1000EEEE","s+":"162"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"563B70548EC3","b":"563B6DBBF000","o":"2989EC3","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_21KeysCollectionManager14PeriodicRunner5startEPNS3_14ServiceContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEENS3_8DurationISt5ratioILl1ELl1000EEEEEUlvE_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"93"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"563B70A30E7F","b":"563B6DBBF000","o":"2E71E7F","s":"execute_native_thread_routine","s+":"F"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"7F88D4722EA5","b":"7F88D471B000","o":"7EA5","s":"start_thread","s+":"C5"}}}
{"t":{"$date":"2021-01-15T03:30:06.020+07:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"monitoring-keys-for-HMAC","msg":" Frame: {frame}","attr":{"frame":{"a":"7F88D444B8DD","b":"7F88D434D000","o":"FE8DD","s":"clone","s+":"6D"}}}
{"t":{"$date":"2021-01-15T09:10:39.116+07:00"},"s":"W", "c":"CONTROL", "id":20698, "ctx":"main","msg":"***** SERVER RESTARTED *****","tags":["startupWarnings"]}
Could you help me to avoid this issue?