I have a mongodB, version 4.4, instance running locally on my machine. I tried to upload 1.2GB file to run from the same local machine. But it is very slow. It took 494 seconds to upload. I tried both bucket based and non bucket APIs.
The mongoc driver is installed using mongo-c-driver-1.23.0.tar.gz and following instructions at mongoc. -DCMAKE_BUILD_TYPE=Release option was selected when running cmake.
Both the code and mongodb are running in Ubuntu 20.04 on a virtual machine.
The code for testing using bucket based is below and is based off the example at mongoc_gridfs_bucket_t — libmongoc 1.23.2. Does anyone have any idea why it takes such a long time to upload? I have used pymongo’s GridFs to upload to the same database and running on the same VM. pymongo performance is good as it takes 5s to upload the same file. So something is strange about mongoc or the way I am using.
int main(int argc, char *argv[])
{
const char *uri_string =
"mongodb://localhost:27017/?appname=new-gridfs-example";
mongoc_client_t *client;
mongoc_database_t *db;
mongoc_stream_t *file_stream;
mongoc_gridfs_bucket_t *bucket;
mongoc_cursor_t *cursor;
bson_t filter;
bool res;
bson_value_t file_id;
bson_error_t error;
const bson_t *doc;
bson_t *opts;
char *str;
time_t start_t, end_t;
mongoc_write_concern_t *write_concern;
mongoc_init ();
/*
write_concern = mongoc_write_concern_new ();
mongoc_write_concern_set_w (write_concern, 1 ); // wtimeoutMS
*/
opts = BCON_NEW("bucketName", "testFs");
//mongoc_write_concern_append (write_concern, opts);
// 1. Make a bucket.
client = mongoc_client_new (uri_string);
db = mongoc_client_get_database (client, "test");
bucket = mongoc_gridfs_bucket_new (db, opts, NULL, &error);
if (!bucket) {
printf ("Error creating gridfs bucket: %s\n", error.message);
return EXIT_FAILURE;
}
// 2. Insert a file.
time(&start_t);
file_stream = mongoc_stream_file_new_for_path ("abc.dat", O_RDONLY, 0);
res = mongoc_gridfs_bucket_upload_from_stream (
bucket, "my-file", file_stream, NULL, &file_id, &error);
if (!res) {
printf ("Error uploading file: %s\n", error.message);
return EXIT_FAILURE;
}
time(&end_t);
printf("Time for uploading %f\n", difftime(end_t, start_t));
mongoc_stream_close (file_stream);
mongoc_stream_destroy (file_stream);
return EXIT_SUCCESS;
}
The non bucket API took 122 seconds which is relatively faster than bucket based but still very slow.
The code is based of mongoc_gridfs_t — libmongoc 1.23.2.
int main(int argc, char *argv[])
{
mongoc_gridfs_t *gridfs;
mongoc_gridfs_file_t *file;
mongoc_gridfs_file_list_t *list;
mongoc_gridfs_file_opt_t opt = {0};
mongoc_client_t *client;
const char *uri_string = "mongodb://127.0.0.1:27017/?appname=gridfs-example";
mongoc_uri_t *uri;
mongoc_stream_t *stream;
bson_t filter;
bson_t opts;
bson_t child;
bson_error_t error;
ssize_t r;
char buf[4096];
mongoc_iovec_t iov;
const char *filename;
const char *command;
bson_value_t id;
time_t start_t, end_t;
/*
if (argc < 2) {
fprintf (stderr, "usage - %s command ...\n", argv[0]);
return EXIT_FAILURE;
}
*/
mongoc_init ();
iov.iov_base = (void *) buf;
iov.iov_len = sizeof buf;
/* connect to localhost client */
uri = mongoc_uri_new_with_error (uri_string, &error);
if (!uri) {
fprintf (stderr,
"failed to parse URI: %s\n"
"error message: %s\n",
uri_string,
error.message);
return EXIT_FAILURE;
}
client = mongoc_client_new_from_uri (uri);
assert (client);
mongoc_client_set_error_api (client, 2);
// grab a gridfs handle in test prefixed by fs
gridfs = mongoc_client_get_gridfs (client, "test", "fs", &error);
assert (gridfs);
command = "write"; //argv[1];
//filename = argv[2];
if (strcmp (command, "read") == 0) {
filename = "hello";
file = mongoc_gridfs_find_one_by_filename (gridfs, filename, &error);
assert (file);
stream = mongoc_stream_gridfs_new (file);
assert (stream);
for (;;) {
r = mongoc_stream_readv (stream, &iov, 1, -1, 0);
assert (r >= 0);
if (r == 0) {
break;
}
if (fwrite (iov.iov_base, 1, r, stdout) != r) {
MONGOC_ERROR ("Failed to write to stdout. Exiting.\n");
exit (1);
}
}
mongoc_stream_destroy (stream);
mongoc_gridfs_file_destroy (file);
} else if (strcmp (command, "list") == 0) {
bson_init (&filter);
bson_init (&opts);
bson_append_document_begin (&opts, "sort", -1, &child);
BSON_APPEND_INT32 (&child, "filename", 1);
bson_append_document_end (&opts, &child);
list = mongoc_gridfs_find_with_opts (gridfs, &filter, &opts);
bson_destroy (&filter);
bson_destroy (&opts);
while ((file = mongoc_gridfs_file_list_next (list))) {
const char *name = mongoc_gridfs_file_get_filename (file);
printf ("%s\n", name ? name : "?");
mongoc_gridfs_file_destroy (file);
}
mongoc_gridfs_file_list_destroy (list);
} else if (strcmp (command, "write") == 0) {
time(&start_t);
stream = mongoc_stream_file_new_for_path ("abc.dat", O_RDONLY, 0);
assert (stream);
opt.filename = filename;
// the driver generates a file_id for you
file = mongoc_gridfs_create_file_from_stream (gridfs, stream, &opt);
assert (file);
id.value_type = BSON_TYPE_INT32;
id.value.v_int32 = 1;
// optional: the following method specifies a file_id of any BSON type
/*if (!mongoc_gridfs_file_set_id (file, &id, &error)) {
fprintf (stderr, "%s\n", error.message);
return EXIT_FAILURE;
}*/
if (!mongoc_gridfs_file_save (file)) {
mongoc_gridfs_file_error (file, &error);
fprintf (stderr, "Could not save: %s\n", error.message);
return EXIT_FAILURE;
}
time(&end_t);
printf("Time for uploading %f\n", difftime(end_t, start_t));
mongoc_gridfs_file_destroy (file);
} else {
fprintf (stderr, "Unknown command");
return EXIT_FAILURE;
}
mongoc_gridfs_destroy (gridfs);
mongoc_uri_destroy (uri);
mongoc_client_destroy (client);
mongoc_cleanup ();
return EXIT_SUCCESS;
}