Mongodb Nodejs Process Out Of Memory

I am trying to insert a million+ records (dummy) using a NodeJS Program in MongoDB collection.But unfortunately my process runs out of memory: This is the code I wrote in JavaScrip

Solution 1:

If your MongoDB server is 2.6 or newer, it would be better to take advantage of using a write commands Bulk API that allow for the execution of bulk insert operations which are simply abstractions on top of the server to make it easy to build bulk operations. These bulk operations come mainly in two flavours:

• Ordered bulk operations. These operations execute all the operation in order and error out on the first write error.
• Unordered bulk operations. These operations execute all the operations in parallel and aggregates up all the errors. Unordered bulk operations do not guarantee order of execution.

Note, for older servers than 2.6 the API will downconvert the operations. However it's not possible to downconvert 100% so there might be some edge cases where it cannot correctly report the right numbers.

In your case, you could implement the Bulk API like this:

varMongoClient = require('mongodb').MongoClient;

MongoClient.connect("mongodb://localhost:27017/course", function(err, db) {
    // Handle errorif(err) throw err;

    // Get the collection and bulk api artefactsvar col = db.collection('students'),
        types = ['exam', 'quiz', 'homework', 'homework'],
        bulk = col.initializeOrderedBulkOp(), // Initialize the Ordered Batch
        counter = 0;

    // Drop the collection
    col.drop();


    // Representing a long loop with 1 Million Recordsfor (var i = 0; i < 1000000; i++) {
        var scores = [],
            class_id = 0,
            record = {};

        // Each student taking 10 classesfor (var class_counter = 0; class_counter < 10; class_counter ++) {

            // Each Class has 4 grades// and each class has 4 gradesfor (var j = 0; j < 4; j++) {
                scores.push({ 'type': types[j], 'score': Math.random()*100 });
            }

            // there are 500 different classes that they can take
            class_id = Math.floor(Math.random() * 501); // get a class id between 0 and 500    

            record['student_id'] = i;
            record['scores'] = scores;
            record['class_id'] = class_id;
        }

        bulk.insert(record);            
        counter++;

        if (counter % 1000 == 0 ) {
            bulk.execute(function(err, result) {  
                // re-initialise batch operation           
                bulk = col.initializeOrderedBulkOp();
            });
        }
    }

    if (counter % 1000 != 0 ){
        bulk.execute(function(err, result) {
            // do something with result
            db.close();
        }); 
    } 
});

-- UPDATE --

Kudos to @MarkusWMahlberg, for generating dummy content you may want to try the package mgenerate.

Solution 2:

I have successfully tested the following code in the environment:

• Processor: Intel(R) Core(TM) i5-3470 CPU @ 3.20GHz 64 bits
• RAM: 4,9 GB
• Operating System: Ubuntu 16.04 LTS
• NodeJS: 4.4.2
• MongoDB: 3.2.1

var mongodb = require('mongodb');

main();

functionmain() {
    doWork(onDoWork);
    functiononDoWork(error, result) {
        if (error) {
            console.error(error);
        }
        elseif (result) {
            console.log(result);
        }
    }
}

functiondoWork(callback) {
    mongodb.MongoClient.connect('mongodb://localhost:27017/course', onConnect);
    functiononConnect(error, db) {
        if (error) returnsetImmediate(callback, error);
        varIMIN = 0;
        varIMAX = 1000000 - 1;
        var i = IMIN;
        varJMIN = 0;
        varJMAX = 10 - 1;
        var j = JMIN;       
        insertOne(db, i, onInsertOne);
        functiononInsertOne(error, result) {
            if (error) {
                db.close();
                setImmediate(callback, error);
            }
            elseif (i > IMAX) {
                db.close();
                setImmediate(callback);
            }
            else {
                i++;
                j++;
                if (j > JMAX) j = JMIN;
                insertOne(db, i, onInsertOne);
            }
        }
    }
}

functioninsertOne(db, studentId, callback) {
    var types = ['exam', 'quiz', 'homework', 'homework'];
    varKMIN = 0;
    varKMAX = types.length - 1;
    var scores = [];
    for (var k = KMIN; k <= KMAX; k++) {
        scores.push({
            type: types[k],
            score: Math.round(Math.random() * 100)
        });
    }
    var record = {
        student_id: studentId,
        scores: scores,
        class_id: Math.floor(Math.random() * 500) + 1
    };
    db.collection('students').insertOne(record, {}, callback);  
}