Hi
We are processing historical quote data from files (millions of rows). I have followed fixed size bucketing (1000 quotes per document) for time series data as per scenario # 3 in MongoDB's New Time Series Collections | MongoDB
so to push data to mongodb I am sending 10000 transactions (bulk.find().upsert().updateone()) between bulk initialization (tried both ordered and non ordered) and bulk.execute() (tried both with write and no write concern), but somehow performance is significant slow (it is pushing 1000 quotes per second)
here is example of generated query which is being pushed to mongodb through java (for clarity I have put 5 transactions here instead of 10000). Any suggestion why it is significant slow?
var bulk=db.quote_date.initializeOrderedBulkOp();
bulk.find({ric:"LCOJ1-V1",utc_date:ISODate("2020-08-13T00:00:00.000000000Z"),nquotes:{$lt:1000}}).upsert().updateOne(
{
$push:{quotes:{utc_date_time:NumberInt("1597337417"),nano_seconds:NumberInt(784325554),ask_price:-0.98,ask_size:NumberInt(4),bid_price:-1.0,bid_size:NumberInt(4),qualifiers:["[BID_TONE]","[ASK_TONE]"]}},
$setOnInsert:{exch_utc_offset:NumberInt(1)},
$min:{quote_first:NumberLong("1597337417784")},
$max:{quote_last:NumberLong("1597337417784")},
$inc :{nquotes:NumberInt(1)}
});
bulk.find({ric:"LCOJ1-V1",utc_date:ISODate("2020-08-13T00:00:00.000000000Z"),nquotes:{$lt:1000}}).upsert().updateOne(
{
$push:{quotes:{utc_date_time:NumberInt("1597337606"),nano_seconds:NumberInt(436207836),ask_price:-0.97,ask_size:NumberInt(4),bid_price:-1.0,bid_size:NumberInt(4)}},
$setOnInsert:{exch_utc_offset:NumberInt(1)},
$min:{quote_first:NumberLong("1597337606436")},
$max:{quote_last:NumberLong("1597337606436")},
$inc :{nquotes:NumberInt(1)}
});
bulk.find({ric:"LCOJ1-V1",utc_date:ISODate("2020-08-13T00:00:00.000000000Z"),nquotes:{$lt:1000}}).upsert().updateOne(
{
$push:{quotes:{utc_date_time:NumberInt("1597337635"),nano_seconds:NumberInt(967713742),ask_price:-0.98,ask_size:NumberInt(4),bid_price:-1.0,bid_size:NumberInt(4)}},
$setOnInsert:{exch_utc_offset:NumberInt(1)},
$min:{quote_first:NumberLong("1597337635967")},
$max:{quote_last:NumberLong("1597337635967")},
$inc :{nquotes:NumberInt(1)}
});
bulk.find({ric:"LCOJ1-V1",utc_date:ISODate("2020-08-13T00:00:00.000000000Z"),nquotes:{$lt:1000}}).upsert().updateOne(
{
$push:{quotes:{utc_date_time:NumberInt("1597337940"),nano_seconds:NumberInt(812651241),ask_price:-0.98,ask_size:NumberInt(4),qualifiers:["[BID_TONE]"]}},
$setOnInsert:{exch_utc_offset:NumberInt(1)},
$min:{quote_first:NumberLong("1597337940812")},
$max:{quote_last:NumberLong("1597337940812")},
$inc :{nquotes:NumberInt(1)}
});
bulk.find({ric:"LCOJ1-V1",utc_date:ISODate("2020-08-13T00:00:00.000000000Z"),nquotes:{$lt:1000}}).upsert().updateOne(
{
$push:{quotes:{utc_date_time:NumberInt("1597337940"),nano_seconds:NumberInt(832562892),ask_price:-0.99,ask_size:NumberInt(4)}},
$setOnInsert:{exch_utc_offset:NumberInt(1)},
$min:{quote_first:NumberLong("1597337940832")},
$max:{quote_last:NumberLong("1597337940832")},
$inc :{nquotes:NumberInt(1)}
});
bulk.execute({w:0});