I have a collection contains about 277492 records, use BulkWrite API to update all records.
codes like this(golang):
client, _ := mongo.Client(....uri....)
tb := client.Database("database_name").Collection("table_name")
cursor := tb.Find({}).Cursor()
var maxBulkSize = 15000
var model DataModelStruct{
ID primitive.ObjectID `bson:"_id"`
UpdateAt uint64 `bson:"update_at"`
}
bulks := make([]DataModelStruct, 0, maxBulkSize)
for cursor.Next(&model) {
bulks = append(bulks, model)
if len(bulks) < maxBulkSize {
continue
}
bm := make([]UpdateOneModel, 0, maxBulkSize)
for _, m := range bulks {
uom := NewUpdateOneModel().
SetFilter(bson.M{"_id":m.ID}).
SetUpdate(bson.M{
"$set": bson.M{
"update_at": time.Now().UnixNano() // this will be changed everytime to ensure this value not equal the value already in doc
}
})
bm = append(bm, uom)
}
res,err := tb.BulkWrite(ctx, bm, ordered=false)
// no err is here
// reset bm、bulks to zero slice
bm = make([]UpdateOneModel, 0, maxBulkSize)
bulks = make([]DataModelStruct, 0, maxBulkSize)
// but got res.MatchedCount not equal res.ModifiedCount there
}
2023-06-08 16:13:52: start migrate basebill, concurrency:20000, online:false
2023-06-08 16:13:59: loop update count: 15000, Matched:14999 Modified:10732
2023-06-08 16:14:07: loop update count: 30000, Matched:14999 Modified:10758
2023-06-08 16:14:13: loop update count: 45000, Matched:14999 Modified:11163
2023-06-08 16:14:14: loop update count: 60000, Matched:14999 Modified:9661
2023-06-08 16:14:19: loop update count: 75000, Matched:15000 Modified:11181
2023-06-08 16:14:22: loop update count: 90000, Matched:15000 Modified:13024
2023-06-08 16:14:24: loop update count: 105000, Matched:15000 Modified:7384
2023-06-08 16:14:28: loop update count: 120000, Matched:15000 Modified:11119
2023-06-08 16:14:31: loop update count: 135000, Matched:14999 Modified:12710
2023-06-08 16:14:35: loop update count: 150000, Matched:14999 Modified:12216
2023-06-08 16:14:36: loop update count: 165000, Matched:15000 Modified:10312
2023-06-08 16:14:40: loop update count: 180000, Matched:15000 Modified:11271
2023-06-08 16:14:44: loop update count: 195000, Matched:15000 Modified:12886
2023-06-08 16:14:47: loop update count: 210000, Matched:15000 Modified:11166
2023-06-08 16:14:50: loop update count: 225000, Matched:15000 Modified:12736
2023-06-08 16:14:51: loop update count: 240000, Matched:15000 Modified:8337
2023-06-08 16:14:54: loop update count: 255000, Matched:15000 Modified:10093
2023-06-08 16:14:59: loop update count: 270000, Matched:15000 Modified:12313
2023-06-08 16:15:02: last update count: 277492, Matched:7492 Modified:6813
2023-06-08 16:15:02: end migrate basebill, online:false, total:277492