Inserting huge data using nodejs SQL DB2

Question

I got lets say 100.000 records in array:

                var eData = { "id": "1001", "type": "Regular" },
                { "id": "1002", "type": "Chocolate" },
                { "id": "1003", "type": "Blueberry" },
                { "id": "1004", "type": "Devil's Food" }

And so on... When I fire the node.js script below

var db = require('/QOpenSys/QIBM/ProdData/OPS/Node6/os400/db2i/lib/db2a');
var DBname = "*LOCAL";
var dbconn = new db.dbconn();
dbconn.conn(DBname);
var sqlA = new db.dbstmt(dbconn);

eData.forEach(function(eRow, i) {
     var sql = "INSERT INTO lib.table VALUES( xx x x x) WITH NONE"

     sqlA.exec(sql, function(rs, err) {
        console.log("Execute Done.");
        console.log(err);
     });
});

The data will be mixed up in DB. Same id and type will be there 10 times, but it will hit the exact number of insertet records.

If I change to execSync, everything turns out right, but seams a bit slow. What am I missing to do async inserts?

What is the fastest way doing huge inserts?

serby · Accepted Answer · 2017-08-17 22:25:14Z

1

There will be a optimal number of async operations to have processing at any one time. The easiest way to limit the number of async operations is with the excellent async.js module.

https://caolan.github.io/async/docs.html#eachLimit

var async = require('async')
var db = require('/QOpenSys/QIBM/ProdData/OPS/Node6/os400/db2i/lib/db2a');
var DBname = "*LOCAL";
var dbconn = new db.dbconn();
dbconn.conn(DBname);
var sqlA = new db.dbstmt(dbconn);

async.eachLimit(eData, 100, function(eRow, cb) {
     var sql = "INSERT INTO lib.table VALUES( xx x x x) WITH NONE"

     sqlA.exec(sql, function(rs, err) {
        console.log("Execute Done.");
        cb(err)
     });
}, function (error) {
    if (error) {
        console.error(error)
    } else {
        console.log('Done')
    }
})

answered Aug 17, 2017 at 22:25

serby

4,3462 gold badges27 silver badges25 bronze badges

Sign up to request clarification or add additional context in comments.

1 Comment

AP87 Over a year ago

Sorry this dosnt help me. Even with only 10 records to insert, it will mix up the data in DB.

Collectives™ on Stack Overflow

Inserting huge data using nodejs SQL DB2

1 Answer 1

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related