Nodejs/Express CSV Upload Download to MySQL/PostgreSQL – Multer, Fast-CSV, Json2Csv, Sequelize

Author: loizenai

Nodejs-Upload-CSV-Files/app/config/db.config.js

@@ -0,0 +1,24 @@
+const env = require('./env.js');
+ 
+const Sequelize = require('sequelize');
+const sequelize = new Sequelize(env.database, env.username, env.password, {
+  host: env.host,
+  dialect: env.dialect,
+  operatorsAliases: false,
+ 
+  pool: {
+    max: env.max,
+    min: env.pool.min,
+    acquire: env.pool.acquire,
+    idle: env.pool.idle
+  }
+});
+ 
+const db = {};
+ 
+db.Sequelize = Sequelize;
+db.sequelize = sequelize;
+ 
+db.Customer = require('../models/customer.model.js')(sequelize, Sequelize);
+ 
+module.exports = db;

Nodejs-Upload-CSV-Files/app/config/env.js

@@ -0,0 +1,15 @@
+const env = {
+  database: 'loizenaidb',
+  username: 'root',
+  password: '12345',
+  host: 'localhost',
+  dialect: 'mysql',
+  pool: {
+    max: 5,
+    min: 0,
+    acquire: 30000,
+    idle: 10000
+  }
+};
+ 
+module.exports = env;

Nodejs-Upload-CSV-Files/app/config/multer.config.js

@@ -0,0 +1,14 @@
+const multer = require('multer');
+
+const storage = multer.diskStorage({
+    destination: (req, file, cb) => {
+       cb(null, __basedir + '/uploads/')
+    },
+    filename: (req, file, cb) => {
+       cb(null, file.fieldname + "-" + Date.now() + "-" + file.originalname)
+    }
+  });   
+   
+const upload = multer({storage: storage});
+
+module.exports = upload;

Nodejs-Upload-CSV-Files/app/controllers/csv.controller.js

@@ -0,0 +1,124 @@
+const stream = require('stream');
+const await = require('await')
+const fs = require('fs');
+const path = require('path');
+
+const db = require('../config/db.config.js');
+const Customer = db.Customer;
+
+const csv = require('fast-csv');
+const Json2csvParser = require('json2csv').Parser;
+
+/**
+ * Upload Single CSV file/ and Import data to MySQL/PostgreSQL database
+ * @param {*} req 
+ * @param {*} res 
+ */
+exports.uploadFile = (req, res) => {
+    try{
+        const customers = [];
+        fs.createReadStream(__basedir + "/uploads/" + req.file.filename)
+            .pipe(csv.parse({ headers: true }))
+            .on('error', error => {
+                console.error(error);
+                throw error.message;
+            })
+            .on('data', row => {
+                customers.push(row);
+                console.log(row);
+            })
+            .on('end', () => {
+                // Save customers to MySQL/PostgreSQL database
+                Customer.bulkCreate(customers).then(() => {
+                    const result = {
+                        status: "ok",
+                        filename: req.file.originalname,
+                        message: "Upload Successfully!",
+                    }
+    
+                    res.json(result);
+                });    
+            });
+    }catch(error){
+        const result = {
+            status: "fail",
+            filename: req.file.originalname,
+            message: "Upload Error! message = " + error.message
+        }
+        res.json(result);
+    }
+}
+
+/** 
+ * Upload multiple Excel Files
+ *  
+ * @param {*} req 
+ * @param {*} res 
+ */
+exports.uploadMultipleFiles = async (req, res) => {
+    const messages = [];
+    
+	for (const file of req.files) {
+        try{
+            // Parsing CSV Files to data array objects
+            const csvParserStream = fs.createReadStream(__basedir + "/uploads/" + file.filename)
+                        .pipe(csv.parse({ headers: true }));
+
+            var end = new Promise(function(resolve, reject) {
+                let customers = [];
+
+                csvParserStream.on('data', object => {
+                    customers.push(object);
+                    console.log(object);
+                });
+                csvParserStream.on('end', () => {
+                    resolve(customers);
+                });
+                csvParserStream.on('error', error => {
+                    console.error(error);
+                    reject
+                }); // or something like that. might need to close `hash`
+            });
+            
+            await (async function() {
+                let customers = await end;
+
+                // save customers to MySQL/PostgreSQL database
+                await Customer.bulkCreate(customers).then(() => {
+                    const result = {
+                        status: "ok",
+                        filename: file.originalname,
+                        message: "Upload Successfully!",
+                    }
+    
+                    messages.push(result);
+                }); 
+            }());
+        }catch(error){
+            console.log(error);
+
+            const result = {
+                status: "fail",
+                filename: file.originalname,				
+                message: "Error -> " + error.message
+            }
+            messages.push(result);
+        }
+	}
+
+	return res.json(messages);
+}
+
+
+exports.downloadFile = (req, res) => {
+    Customer.findAll({attributes: ['id', 'name', 'address', 'age']}).then(objects => {
+        const jsonCustomers = JSON.parse(JSON.stringify(objects));
+        const csvFields = ['Id', 'Name', 'Address', 'Age'];
+        const json2csvParser = new Json2csvParser({ csvFields });
+        const csvData = json2csvParser.parse(jsonCustomers);
+
+        res.setHeader('Content-disposition', 'attachment; filename=customers.csv');
+        res.set('Content-Type', 'text/csv');
+        res.status(200).end(csvData);
+    });
+}

Nodejs-Upload-CSV-Files/app/models/customer.model.js

@@ -0,0 +1,20 @@
+module.exports = (sequelize, Sequelize) => {
+	const Customer = sequelize.define('customer', {	
+	  id: {
+            type: Sequelize.INTEGER,
+            autoIncrement: true,
+            primaryKey: true
+      },
+	  name: {
+			type: Sequelize.STRING
+	  },
+	  address: {
+			type: Sequelize.STRING
+	  },
+	  age: {
+			type: Sequelize.INTEGER
+	  }
+	});
+	
+	return Customer;
+}

Nodejs-Upload-CSV-Files/app/routers/excel.router.js

@@ -0,0 +1,19 @@
+let express = require('express');
+let router = express.Router();
+let upload = require('../config/multer.config.js');
+ 
+const csvWorker = require('../controllers/csv.controller.js');
+
+let path = __basedir + '/views/';
+
+router.get('/', (req,res) => {
+    console.log("__basedir" + __basedir);
+    res.sendFile(path + "index.html");
+});
+
+router.post('/api/file/upload', upload.single("file"), csvWorker.uploadFile);
+router.post('/api/file/multiple/upload', upload.array('files', 4), csvWorker.uploadMultipleFiles);
+
+router.get('/api/file', csvWorker.downloadFile);
+
+module.exports = router;