I found some error logs in mongod log file ，the errormsg as follows
{“t”:{“$date”:“2023-08-27T15:12:05.257+08:00”},“s”:“W”, “c”:“SHARDING”, “id”:23777, “ctx”:“MoveChunk”,“msg”:“Error while doing moveChunk”,“attr”:{“error”:“OperationFailed:** Data transfer error: ExceededTimeLimit: Failed to delete orphaned** mobile_prd_db.webservicemes range [{ _id: ObjectId(‘5f405f5f7c17949ae0ee09e1’) }, { _id: ObjectId(‘5f4093123141f4e75bbda659’) }) :: caused by :: operation exceeded time limit”}}
{“t”:{“$date”:“2023-08-27T15:12:05.257+08:00”},“s”:“I”, “c”:“SHARDING”, “id”:22080, “ctx”:“MoveChunk”,“msg”:“About to log metadata event”,“attr”:{“namespace”:“changelog”,“event”:{“_id”:“hzcmongo07:9421-2023-08-27T15:12:05.257+08:00-64eaf7459b7f050c49fccb26”,“server”:“hzcmongo07:9421”,“shard”:“rep_shard1”,“clientAddr”:“”,“time”:{“$date”:“2023-08-27T07:12:05.257Z”},“what”:“moveChunk.from”,“ns”:“mobile_prd_db.webservicemes”,“details”:{“min”:{“_id”:{“$oid”:“5f405f5f7c17949ae0ee09e1”}},“max”:{“_id”:{“$oid”:“5f4093123141f4e75bbda659”}},“step 1 of 6”:0,“step 2 of 6”:3,“step 3 of 6”:17,“to”:“rep_shard3”,“from”:“rep_shard1”,“note”:“aborted”}}}}
the mongoserver is shard mode,has 3 shards， Maybe a problem with my parameter configuration
one of the mongod parameters as follows：
{
"AlwaysRecordTraffic" : "",
"KeysRotationIntervalSec" : 7776000,
"ShardingTaskExecutorPoolHostTimeoutMS" : 600000,
"ShardingTaskExecutorPoolMaxConnecting" : 2,
"ShardingTaskExecutorPoolMaxSize" : 32767,
"ShardingTaskExecutorPoolMaxSizeForConfigServers" : -1,
"ShardingTaskExecutorPoolMinSize" : 1,
"ShardingTaskExecutorPoolMinSizeForConfigServers" : -1,
"ShardingTaskExecutorPoolRefreshRequirementMS" : 60000,
"ShardingTaskExecutorPoolRefreshTimeoutMS" : 20000,
"ShardingTaskExecutorPoolReplicaSetMatching" : "automatic",
"TransactionRecordMinimumLifetimeMinutes" : 30,
"adaptiveServiceExecutorIdlePctThreshold" : 60,
"adaptiveServiceExecutorMaxQueueLatencyMicros" : 500,
"adaptiveServiceExecutorRecursionLimit" : 8,
"adaptiveServiceExecutorReservedThreads" : -1,
"adaptiveServiceExecutorRunTimeJitterMillis" : 500,
"adaptiveServiceExecutorRunTimeMillis" : 5000,
"adaptiveServiceExecutorStuckThreadTimeoutMillis" : 250,
"allowRolesFromX509Certificates" : true,
"allowSecondaryReadsDuringBatchApplication" : true,
"assertStableTimestampEqualsAppliedThroughOnRecovery" : false,
"authSchemaVersion" : 5,
"authenticationMechanisms" : [
"MONGODB-X509",
"SCRAM-SHA-1",
"SCRAM-SHA-256"
],
"authorizationManagerCacheSize" : 100,
"authorizationManagerPinnedUsers" : [ ],
"authorizationManagerPinnedUsersRefreshIntervalMillis" : NumberLong(1000),
"awsEC2InstanceMetadataUrl" : "http://169.254.169.254",
"awsECSInstanceMetadataUrl" : "http://169.254.170.2",
"bgSyncOplogFetcherBatchSize" : 13981010,
"cachePressureThreshold" : 95,
"checkCachePressurePeriodSeconds" : 5,
"clientCursorMonitorFrequencySecs" : 4,
"cloudFreeMonitoringEndpointURL" : "https://cloud.mongodb.com/freemonitoring/mongo",
"clusterAuthMode" : "keyFile",
"collectionBulkLoaderBatchSizeInBytes" : 262144,
"collectionClonerBatchSize" : 0,
"collectionClonerUsesExhaust" : true,
"connPoolMaxConnsPerHost" : 200,
"connPoolMaxInUseConnsPerHost" : 2147483647,
"connPoolMaxShardedConnsPerHost" : 200,
"connPoolMaxShardedInUseConnsPerHost" : 2147483647,
"connectTimeoutMs" : 10000,
"createRollbackDataFiles" : true,
"cursorTimeoutMillis" : NumberLong(600000),
"dbCheckCollectionTryLockMaxAttempts" : 5,
"dbCheckCollectionTryLockMaxBackoffMillis" : 60000,
"dbCheckCollectionTryLockMinBackoffMillis" : 10,
"dbCheckCollectionTryLockTimeoutMillis" : 10,
"dbCheckHealthLogEveryNBatches" : 25,
"diagnosticDataCollectionDirectoryPath" : "/u02/mongodb4max/data/9422data/diagnostic.data",
"diagnosticDataCollectionDirectorySizeMB" : 200,
"diagnosticDataCollectionEnableLatencyHistograms" : false,
"diagnosticDataCollectionEnabled" : true,
"diagnosticDataCollectionFileSizeMB" : 10,
"diagnosticDataCollectionPeriodMillis" : 1000,
"diagnosticDataCollectionSamplesPerChunk" : 300,
"diagnosticDataCollectionSamplesPerInterimUpdate" : 10,
"diagnosticDataCollectionVerboseTCMalloc" : false,
"disableJavaScriptJIT" : true,
"disableLogicalSessionCacheRefresh" : false,
"disableNonSSLConnectionLogging" : false,
"disableNonTLSConnectionLogging" : false,
"disableResumableRangeDeleter" : false,
"disabledSecureAllocatorDomains" : [ ],
"enableElectionHandoff" : true,
"enableFinerGrainedCatalogCacheRefresh" : false,
"enableFlowControl" : true,
"enableHybridIndexBuilds" : true,
"enableIndexBuildCommitQuorum" : true,
"enableLocalhostAuthBypass" : true,
"enableOverrideClusterChainingSetting" : false,
"enableReconfigRollbackCommittedWritesCheck" : true,
"enableSafeReplicaSetReconfig" : true,
"enableSearchMeta" : true,
"enableShardedIndexConsistencyCheck" : true,
"enableTestCommands" : false,
"enableTimeoutOfInactiveSessionCursors" : false,
"enableTwoPhaseIndexBuild" : true,
"enforceUserClusterSeparation" : true,
"featureCompatibilityVersion" : {
"version" : "4.4"
},
"featureFlagHealthMonitoring" : true,
"findChunksOnConfigTimeoutMS" : 900000,
"flowControlDecayConstant" : 0.5,
"flowControlFudgeFactor" : 0.95,
"flowControlMaxSamples" : 1000000,
"flowControlMinTicketsPerSecond" : 100,
"flowControlSamplePeriod" : 1000,
"flowControlTargetLagSeconds" : 10,
"flowControlThresholdLagPercentage" : 0.5,
"flowControlTicketAdderConstant" : 1000,
"flowControlTicketMultiplierConstant" : 1.05,
"flowControlWarnThresholdSeconds" : 10,
"forceRollbackViaRefetch" : false,
"globalConnPoolIdleTimeoutMinutes" : 2147483647,
"heapProfilingEnabled" : false,
"heapProfilingSampleIntervalBytes" : NumberLong(262144),
"heartBeatFrequencyMs" : 10000,
"honorSystemUmask" : false,
"initialSyncOplogBuffer" : "collection",
"initialSyncOplogBufferPeekCacheSize" : 10000,
"initialSyncOplogFetcherBatchSize" : 13981010,
"initialSyncSourceReadPreference" : "",
"initialSyncTransientErrorRetryPeriodSeconds" : 86400,
"internalDocumentSourceCursorBatchSizeBytes" : 4194304,
"internalDocumentSourceGroupMaxMemoryBytes" : NumberLong(104857600),
"internalDocumentSourceLookupCacheSizeBytes" : 104857600,
"internalGeoNearQuery2DMaxCoveringCells" : 16,
"internalGeoPredicateQuery2DMaxCoveringCells" : 16,
"internalIndexBuildBulkLoadYieldIterations" : 1000,
"internalInsertMaxBatchSize" : 500,
"internalLookupStageIntermediateDocumentMaxSizeBytes" : NumberLong(104857600),
"internalPipelineLengthLimit" : 2147483647,
"internalProhibitShardOperationRetry" : false,
"internalQueryAllowShardedLookup" : false,
"internalQueryAlwaysMergeOnPrimaryShard" : false,
"internalQueryCacheDisableInactiveEntries" : false,
"internalQueryCacheEvictionRatio" : 10,
"internalQueryCacheMaxSizeBytesBeforeStripDebugInfo" : NumberLong(536870912),
"internalQueryCacheSize" : 5000,
"internalQueryCacheWorksGrowthCoefficient" : 2,
"internalQueryDesugarWhereToFunction" : false,
"internalQueryDisableExchange" : false,
"internalQueryEnumerationMaxIntersectPerAnd" : 3,
"internalQueryEnumerationMaxOrSolutions" : 10,
"internalQueryEnumerationPreferLockstepOrEnumeration" : false,
"internalQueryExecYieldIterations" : 1000,
"internalQueryExecYieldPeriodMS" : 10,
"internalQueryExplainSizeThresholdBytes" : 10485760,
"internalQueryFacetBufferSizeBytes" : 104857600,
"internalQueryFacetMaxOutputDocSizeBytes" : NumberLong(104857600),
"internalQueryForceIntersectionPlans" : false,
"internalQueryGlobalProfilingFilter" : false,
"internalQueryIgnoreUnknownJSONSchemaKeywords" : false,
"internalQueryJavaScriptFnTimeoutMillis" : 60000,
"internalQueryJavaScriptHeapSizeLimitMB" : 100,
"internalQueryMaxAddToSetBytes" : NumberLong(104857600),
"internalQueryMaxBlockingSortMemoryUsageBytes" : 104857600,
"internalQueryMaxJsEmitBytes" : 104857600,
"internalQueryMaxPushBytes" : 104857600,
"internalQueryMaxRangeBytes" : 104857600,
"internalQueryMaxScansToExplode" : 200,
"internalQueryPlanEvaluationCollFraction" : 0.3,
"internalQueryPlanEvaluationMaxResults" : 101,
"internalQueryPlanEvaluationWorks" : 10000,
"internalQueryPlanOrChildrenIndependently" : true,
"internalQueryPlannerEnableHashIntersection" : false,
"internalQueryPlannerEnableIndexIntersection" : true,
"internalQueryPlannerGenerateCoveredWholeIndexScans" : false,
"internalQueryPlannerMaxIndexedSolutions" : 64,
"internalQueryProhibitBlockingMergeOnMongoS" : false,
"internalQueryProhibitMergingOnMongoS" : false,
"internalQueryS2GeoCoarsestLevel" : 0,
"internalQueryS2GeoFinestLevel" : 23,
"internalQueryS2GeoMaxCells" : 20,
"internalValidateFeaturesAsMaster" : true,
"javascriptProtection" : false,
"journalCommitInterval" : 100,
"jsHeapLimitMB" : 1100,
"loadRoutingTableOnStartup" : true,
"localLogicalSessionTimeoutMinutes" : 30,
"localThresholdMs" : 15,
"logComponentVerbosity" : {
"verbosity" : 0,
"accessControl" : {
"verbosity" : -1
},
"command" : {
"verbosity" : -1
},
"control" : {
"verbosity" : -1
},
"executor" : {
"verbosity" : -1
},
"geo" : {
"verbosity" : -1
},
"index" : {
"verbosity" : -1
},
"network" : {
"verbosity" : -1,
"asio" : {
"verbosity" : -1
},
"bridge" : {
"verbosity" : -1
},
"connectionPool" : {
"verbosity" : -1
}
},
"processHealth" : {
"verbosity" : -1
},
"query" : {
"verbosity" : -1
},
"replication" : {
"verbosity" : -1,
"election" : {
"verbosity" : -1
},
"heartbeats" : {
"verbosity" : -1
},
"initialSync" : {
"verbosity" : -1
},
"rollback" : {
"verbosity" : -1
}
},
"sharding" : {
"verbosity" : -1,
"rangeDeleter" : {
"verbosity" : -1
},
"shardingCatalogRefresh" : {
"verbosity" : -1
},
"migration" : {
"verbosity" : -1
}
},
"storage" : {
"verbosity" : -1,
"recovery" : {
"verbosity" : -1
},
"journal" : {
"verbosity" : -1
}
},
"write" : {
"verbosity" : -1
},
"ftdc" : {
"verbosity" : -1
},
"tracking" : {
"verbosity" : -1
},
"transaction" : {
"verbosity" : -1
},
"test" : {
"verbosity" : -1
}
},
"logLevel" : 0,
"logicalSessionRefreshMillis" : 300000,
"maxAcceptableLogicalClockDriftSecs" : NumberLong(31536000),
"maxBSONDepth" : 200,
"maxCatchUpPercentageBeforeBlockingWrites" : 10,
"maxIndexBuildDrainBatchSize" : 1000,
"maxIndexBuildDrainMemoryUsageMegabytes" : 16,
"maxIndexBuildMemoryUsageMegabytes" : 200,
"maxLogSizeKB" : 10,
"maxNumActiveUserIndexBuilds" : 3,
"maxNumberOfTransactionOperationsInSingleOplogEntry" : 2147483647,
"maxOplogTruncationPointsAfterStartup" : NumberLong(100),
"maxOplogTruncationPointsDuringStartup" : NumberLong(100),
"maxSessions" : 1000000,
"maxSyncSourceLagSecs" : 30,
"maxTargetSnapshotHistoryWindowInSeconds" : 5,
"maxTimeMSForHedgedReads" : 150,
"maxTransactionLockRequestTimeoutMillis" : 5,
"maxValidateMBperSec" : 0,
"maxValidateMemoryUsageMB" : 200,
"migrateCloneInsertionBatchDelayMS" : 0,
"migrateCloneInsertionBatchSize" : 0,
"migrationLockAcquisitionMaxWaitMS" : 500,
"minMillisBetweenSnapshotWindowDec" : 500,
"minMillisBetweenSnapshotWindowInc" : 500,
"minNumChunksForSessionsCollection" : 1024,
"minOplogTruncationPoints" : NumberLong(10),
"mirrorReads" : {
"samplingRate" : 0.01,
"maxTimeMS" : 1000
},
"mrEnableSingleReduceOptimization" : false,
"notablescan" : false,
"numInitialSyncAttempts" : 10,
"numInitialSyncConnectAttempts" : 10,
"numInitialSyncOplogFindAttempts" : 3,
"ocspCacheSize" : 300,
"ocspEnabled" : true,
"ocspStaplingRefreshPeriodSecs" : NumberLong(-1),
"ocspValidationRefreshPeriodSecs" : NumberLong(-1),
"opensslCipherConfig" : "",
"opensslDiffieHellmanParameters" : "",
"oplogFetcherInitialSyncMaxFetcherRestarts" : 10,
"oplogFetcherSteadyStateMaxFetcherRestarts" : 1,
"oplogFetcherUsesExhaust" : true,
"oplogInitialFindMaxSeconds" : 60,
"oplogNetworkTimeoutBufferSeconds" : 5,
"oplogRetriedFindMaxSeconds" : 2,
"oplogSamplingLogIntervalSeconds" : 10,
"oplogTruncationPointSizeMB" : 0,
"opportunisticSecondaryTargeting" : false,
"orphanCleanupDelaySecs" : 900,
"periodicNoopIntervalSecs" : 10,
"priorityTakeoverFreshnessWindowSeconds" : 2,
"processUmask" : 63,
"quiet" : false,
"rangeDeleterBatchDelayMS" : 20,
"rangeDeleterBatchSize" : 2147483647,
"readHedgingMode" : "on",
"receiveChunkWaitForRangeDeleterTimeoutMS" : 10000,
"recoverFromOplogAsStandalone" : false,
"recoverToOplogTimestamp" : "",
"replBatchLimitBytes" : 104857600,
"replBatchLimitOperations" : 5000,
"replElectionTimeoutOffsetLimitFraction" : 0.15,
"replWriterMinThreadCount" : 0,
"replWriterThreadCount" : 16,
"replicaSetMonitorProtocol" : "streamable",
"reportOpWriteConcernCountersInServerStatus" : false,
"reservedServiceExecutorRecursionLimit" : 8,
"rewriteStateChangeErrors" : true,
"roleGraphInvalidationIsFatal" : false,
"rollbackRemoteOplogQueryBatchSize" : 2000,
"rollbackTimeLimitSecs" : 86400,
"saslHostName" : "hzcmongo08",
"saslServiceName" : "mongodb",
"saslauthdPath" : "",
"scramIterationCount" : 10000,
"scramSHA256IterationCount" : 15000,
"scriptingEngineInterruptIntervalMS" : 1000,
"shardedConnPoolIdleTimeoutMinutes" : 2147483647,
"shardedIndexConsistencyCheckIntervalMS" : 600000,
"shouldMultiDocTxnCreateCollectionAndIndexes" : true,
"skipCorruptDocumentsWhenCloning" : false,
"skipShardingConfigurationChecks" : false,
"skipShellCursorFinalize" : false,
"snapshotWindowAdditiveIncreaseSeconds" : 2,
"snapshotWindowMultiplicativeDecrease" : 0.75,
"sslMode" : "disabled",
"sslWithholdClientCertificate" : false,
"startupAuthSchemaValidation" : true,
"startupRecoveryForRestore" : false,
"storeFindAndModifyImagesInSideCollection" : false,
"suppressNoTLSPeerCertificateWarning" : false,
"syncdelay" : 60,
"synchronousServiceExecutorRecursionLimit" : 8,
"takeUnstableCheckpointOnShutdown" : false,
"taskExecutorPoolSize" : 1,
"tcmallocAggressiveMemoryDecommit" : 0,
"tcmallocMaxTotalThreadCacheBytes" : NumberLong(1073741824),
"tcmallocReleaseRate" : 1,
"tcpFastOpenClient" : true,
"tcpFastOpenQueueSize" : 1024,
"tcpFastOpenServer" : true,
"testingDiagnosticsEnabled" : false,
"testingSnapshotBehaviorInIsolation" : false,
"tlsCATrusts" : null,
"tlsMode" : "disabled",
"tlsOCSPStaplingTimeoutSecs" : -1,
"tlsOCSPVerifyTimeoutSecs" : 5,
"tlsWithholdClientCertificate" : false,
"tlsX509ExpirationWarningThresholdDays" : 30,
"traceExceptions" : false,
"traceWriteConflictExceptions" : false,
"trafficRecordingDirectory" : "",
"transactionLifetimeLimitSeconds" : 60,
"transactionSizeLimitBytes" : NumberLong("9223372036854775807"),
"ttlMonitorEnabled" : true,
"ttlMonitorSleepSecs" : 60,
"useReadOnceCursorsForIndexBuilds" : true,
"waitForSecondaryBeforeNoopWriteMS" : 10,
"waitForStepDownOnNonCommandShutdown" : true,
"warmMinConnectionsInShardingTaskExecutorPoolOnStartup" : true,
"warmMinConnectionsInShardingTaskExecutorPoolOnStartupWaitMS" : 2000,
"watchdogPeriodSeconds" : -1,
"wiredTigerConcurrentReadTransactions" : 128,
"wiredTigerConcurrentWriteTransactions" : 128,
"wiredTigerCursorCacheSize" : -100,
"wiredTigerEngineRuntimeConfig" : "",
"wiredTigerFileHandleCloseIdleTime" : 100000,
"wiredTigerFileHandleCloseMinimum" : 250,
"wiredTigerFileHandleCloseScanInterval" : 10,
"wiredTigerMaxCacheOverflowSizeGB" : 0,
"wiredTigerPrefixSearchForUniqueIndexes" : false,
"wiredTigerSessionCloseIdleTimeSecs" : 300,
"wiredTigerSkipTableLoggingChecksOnStartup" : false,
"writePeriodicNoops" : true,
"ok" : 1,
"$gleStats" : {
"lastOpTime" : Timestamp(0, 0),
"electionId" : ObjectId("7fffffff000000000000002b")
},
"lastCommittedOpTime" : Timestamp(1693122110, 2),
"$configServerState" : {
"opTime" : {
"ts" : Timestamp(1693122114, 3),
"t" : NumberLong(41)
}
},
"$clusterTime" : {
"clusterTime" : Timestamp(1693122114, 3),
"signature" : {
"hash" : BinData(0,"QEop+7e9EUubuXgRGGEXOhjEW5k="),
"keyId" : NumberLong("7216528265488367622")
}
},
"operationTime" : Timestamp(1693122110, 2)
}
````Preformatted text`
anyone can help me ? thank you!!!
i try to change some parameters , like **ShardingTaskExecutorPoolHostTimeoutMS **but doest work