"Invariant failure maxWireVersion" error while adding shard

Hi Team,

Am getting below error on primary config server while adding shard .

Appreciate your help on this!

Log trace

2020-04-05T10:53:15.910+0300 F -        [conn2] Invariant failure maxWireVersion == WireVersion::LATEST_WIRE_VERSION src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp 350
2020-04-05T10:53:15.910+0300 F -        [conn2] 

***aborting after invariant() failure


2020-04-05T10:53:15.949+0300 F -        [conn2] Got signal: 6 (Aborted).
 0x556f9539aea1 0x556f9539a0b9 0x556f9539a59d 0x7feaec7595e0 0x7feaec3bc1f7 0x7feaec3bd8e8 0x556f93976ef8 0x556f93fe8848 0x556f93fec60b 0x556f93c08ca8 0x556f94dff469 0x556f93a2c18a 0x556f93a2da79 0x556f93a2e9c1 0x556f93a1a12a 0x556f93a26d8a 0x556f93a21a37 0x556f93a25251 0x556f94be8472 0x556f93a1fc20 0x556f93a22d65 0x556f93a21177 0x556f93a21abd 0x556f93a25251 0x556f94be89d5 0x556f952f1f44 0x7feaec751e25 0x7feaec47f34d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"556F92F56000","o":"2444EA1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"556F92F56000","o":"24440B9"},{"b":"556F92F56000","o":"244459D"},{"b":"7FEAEC74A000","o":"F5E0"},{"b":"7FEAEC387000","o":"351F7","s":"gsignal"},{"b":"7FEAEC387000","o":"368E8","s":"abort"},{"b":"556F92F56000","o":"A20EF8","s":"_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j"},{"b":"556F92F56000","o":"1092848","s":"_ZN5mongo22ShardingCatalogManager20_validateHostAsShardEPNS_16OperationContextESt10shared_ptrINS_21RemoteCommandTargeterEEPKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_16ConnectionStringE"},{"b":"556F92F56000","o":"109660B","s":"_ZN5mongo22ShardingCatalogManager8addShardEPNS_16OperationContextEPKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_16ConnectionStringEx"},{"b":"556F92F56000","o":"CB2CA8"},{"b":"556F92F56000","o":"1EA9469","s":"_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_19CommandReplyBuilderE"},{"b":"556F92F56000","o":"AD618A"},{"b":"556F92F56000","o":"AD7A79"},{"b":"556F92F56000","o":"AD89C1","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE"},{"b":"556F92F56000","o":"AC412A","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE"},{"b":"556F92F56000","o":"AD0D8A","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},{"b":"556F92F56000","o":"ACBA37","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"556F92F56000","o":"ACF251"},{"b":"556F92F56000","o":"1C92472","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE"},{"b":"556F92F56000","o":"AC9C20","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE"},{"b":"556F92F56000","o":"ACCD65","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"556F92F56000","o":"ACB177","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},{"b":"556F92F56000","o":"ACBABD","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"556F92F56000","o":"ACF251"},{"b":"556F92F56000","o":"1C929D5"},{"b":"556F92F56000","o":"239BF44"},{"b":"7FEAEC74A000","o":"7E25"},{"b":"7FEAEC387000","o":"F834D","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.12", "gitVersion" : "5776e3cbf9e7afe86e6b29e22520ffb6766e95d4", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-693.el7.x86_64", "version" : "#1 SMP Tue Aug 22 21:09:27 UTC 2017", "machine" : "x86_64" }, "somap" : [ { "b" : "556F92F56000", "elfType" : 3, "buildId" : "EF47E1B8B5FC85C7DF8A0916362ACC8238487CB2" }, { "b" : "7FFFC5252000", "elfType" : 3, "buildId" : "7FB8E16CEA1B913E2703A6E4159FB468CD1E3507" }, { "b" : "7FEAEDB77000", "path" : "/lib64/libcurl.so.4", "elfType" : 3, "buildId" : "CE3116A1A44937EC00E131632BFDE144772F83D7" }, { "b" : "7FEAED95D000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "FF4E72F4E574E143330FB3C66DB51613B0EC65EA" }, { "b" : "7FEAED4FC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "BC0AE9CA0705BEC1F0C0375AAD839843BB219CB1" }, { "b" : "7FEAED28A000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "ED0AC7DEB91A242C194B3DEF27A215F41CE43116" }, { "b" : "7FEAED086000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "8CC796BA7CA23193AD753D8625018B61264724BE" }, { "b" : "7FEAECE7E000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "5B629F64AC6EA7AAC602BE56ED834BB6398C72AC" }, { "b" : "7FEAECB7C000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "5FAABA77B1848347CEC4B0CE7B31811D7D00D2FA" }, { "b" : "7FEAEC966000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "361D73E3AA2ACE6AF32B03D1B74A22E1FF68AB2D" }, { "b" : "7FEAEC74A000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B8FBCA68CA56E79556BF7884DACF89504096ADEB" }, { "b" : "7FEAEC387000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "C3F28802314AF4EE866BF8D2E1B506B7BBF34CF6" }, { "b" : "7FEAEDDE0000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "962B8EEE329A2C57184AF2780756ED2035DEAAC0" }, { "b" : "7FEAEC154000", "path" : "/lib64/libidn.so.11", "elfType" : 3, "buildId" : "2B77BBEFFF65E94F3E0B71A4E89BEB68C4B476C5" }, { "b" : "7FEAEBF2A000", "path" : "/lib64/libssh2.so.1", "elfType" : 3, "buildId" : "C4EDF92922B4FE091F9A855C50343288AAD3243D" }, { "b" : "7FEAEBCDE000", "path" : "/lib64/libssl3.so", "elfType" : 3, "buildId" : "A5EC8F3A4BED5873F94B4418BF732FC208DD0C55" }, { "b" : "7FEAEBAB7000", "path" : "/lib64/libsmime3.so", "elfType" : 3, "buildId" : "007CD03E03B51795E5499A88CD670B85061FA226" }, { "b" : "7FEAEB78D000", "path" : "/lib64/libnss3.so", "elfType" : 3, "buildId" : "F5AC0CD5031E4F8A1DDF38CB9EEDCE3D2B12FCA2" }, { "b" : "7FEAEB560000", "path" : "/lib64/libnssutil3.so", "elfType" : 3, "buildId" : "BE353742BC2074F126BB55F42B79E8C4FBE51CD5" }, { "b" : "7FEAEB35C000", "path" : "/lib64/libplds4.so", "elfType" : 3, "buildId" : "CFFD213A7908702160E6EAA8F3F57BCBF906AF94" }, { "b" : "7FEAEB157000", "path" : "/lib64/libplc4.so", "elfType" : 3, "buildId" : "76D4FA4D9CF5FD577C48D3C196AE734A7F9E6CAD" }, { "b" : "7FEAEAF19000", "path" : "/lib64/libnspr4.so", "elfType" : 3, "buildId" : "7B54DBF79ECEB3E7BEF71224EC5EFF3D1B425FA1" }, { "b" : "7FEAEACCC000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "DA322D74F55A0C4293085371A8D0E94B5962F5E7" }, { "b" : "7FEAEA9E4000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "B69E63024D408E400401EEA6815317BDA38FB7C2" }, { "b" : "7FEAEA7B1000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "A48639BF901DB554479BFAD114CB354CF63D7D6E" }, { "b" : "7FEAEA5AD000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "A3832734347DCA522438308C9F08F45524C65C9B" }, { "b" : "7FEAEA39E000", "path" : "/lib64/liblber-2.4.so.2", "elfType" : 3, "buildId" : "38C80306EF7534FF3CD84E946F39806563DE2F3F" }, { "b" : "7FEAEA14A000", "path" : "/lib64/libldap-2.4.so.2", "elfType" : 3, "buildId" : "05BEC066BCE8D1487506C628B4DE39DB79743777" }, { "b" : "7FEAE9F34000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "EA8E45DC8E395CC5E26890470112D97A1F1E0B65" }, { "b" : "7FEAE9D26000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "6FDF5B013FD2739D304CFB9D723DCBC149EE03C9" }, { "b" : "7FEAE9B22000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7FEAE9905000", "path" : "/lib64/libsasl2.so.3", "elfType" : 3, "buildId" : "2936CB6F2025214EC2687205007D819060CE5620" }, { "b" : "7FEAE96DE000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "A88379F56A51950A33198890D37F5F8AEE71F8B4" }, { "b" : "7FEAE94A7000", "path" : "/lib64/libcrypt.so.1", "elfType" : 3, "buildId" : "27C3D04725E31259F87EB3EA8478CF65A0D59568" }, { "b" : "7FEAE9245000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "9CA3D11F018BEEB719CDB34BE800BF1641350D0A" }, { "b" : "7FEAE9042000", "path" : "/lib64/libfreebl3.so", "elfType" : 3, "buildId" : "60C388B53B5D33B7E48DFBFB51E5D8429743BEE8" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x556f9539aea1]
 mongod(+0x24440B9) [0x556f9539a0b9]
 mongod(+0x244459D) [0x556f9539a59d]
 libpthread.so.0(+0xF5E0) [0x7feaec7595e0]
 libc.so.6(gsignal+0x37) [0x7feaec3bc1f7]
 libc.so.6(abort+0x148) [0x7feaec3bd8e8]
 mongod(_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j+0x0) [0x556f93976ef8]
 mongod(_ZN5mongo22ShardingCatalogManager20_validateHostAsShardEPNS_16OperationContextESt10shared_ptrINS_21RemoteCommandTargeterEEPKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_16ConnectionStringE+0xAE8) [0x556f93fe8848]
 mongod(_ZN5mongo22ShardingCatalogManager8addShardEPNS_16OperationContextEPKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_16ConnectionStringEx+0x30B) [0x556f93fec60b]
 mongod(+0xCB2CA8) [0x556f93c08ca8]
 mongod(_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_19CommandReplyBuilderE+0xD9) [0x556f94dff469]
 mongod(+0xAD618A) [0x556f93a2c18a]
 mongod(+0xAD7A79) [0x556f93a2da79]
 mongod(_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE+0x3D1) [0x556f93a2e9c1]
 mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x3A) [0x556f93a1a12a]
 mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0xBA) [0x556f93a26d8a]
 mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x97) [0x556f93a21a37]
 mongod(+0xACF251) [0x556f93a25251]
 mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE+0x1A2) [0x556f94be8472]
 mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE+0x150) [0x556f93a1fc20]
 mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0xB55) [0x556f93a22d65]
 mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x357) [0x556f93a21177]
 mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x11D) [0x556f93a21abd]
 mongod(+0xACF251) [0x556f93a25251]
 mongod(+0x1C929D5) [0x556f94be89d5]
 mongod(+0x239BF44) [0x556f952f1f44]
 libpthread.so.0(+0x7E25) [0x7feaec751e25]
 libc.so.6(clone+0x6D) [0x7feaec47f34d]
-----  END BACKTRACE  -----

Please check the extract maxWireVersion section in below link

Looks like some mismatch in binary version of node being added and cluster’s featureCompatibilityVersion