feat(users, uploads, lib): init uploadx example with gridfs & stream ✨

README.md

@@ -36,7 +36,8 @@ Our stack node is actually in Beta.
 | Testing |  [Jest](https://github.com/facebook/jest) & [SuperTest](https://github.com/visionmedia/supertest) (Coverage & Watch) <br> *example of mocha with gulp available*
 | Linter  | [ESLint](https://github.com/eslint/eslint) ecmaVersion 10 (2019)
 | Security | JWT Stateless - [passport-jwt](https://github.com/themikenicholson/passport-jwt) <br> Passwords: [bcrypt](https://en.wikipedia.org/wiki/Bcrypt) - [zxcvbn](https://github.com/dropbox/zxcvbn) <br> DataBases options available (auth, ssl ..) <br> [SSL](https://github.com/weareopensource/Node/blob/master/WIKI.md#SSL) Express / Reverse Proxy (must be activated, otherwise => plain text password)
-| API | Default answer wrapper (helper) : [jsend](https://github.com/omniti-labs/jsend) like : status, message, data or error <br>  Default error handling (helper) : formatted by the controller, Custom ES6 errors for other layers
+| API | Default answer wrapper (helper) : [jsend](https://github.com/omniti-labs/jsend) like : status, message, data or error <br>  Default errors handling (helper) : formatted by the controller, Custom ES6 errors for other layers
+| Upload | Example : [Mongo gridfs](https://docs.mongodb.com/manual/core/gridfs/) - [mongoose-gridfs](https://github.com/lykmapipo/mongoose-gridfs) - [Multer](https://github.com/expressjs/multer) <br> Avatar stream example available (could catch all contentType)
 | Logs | [winston](https://github.com/winstonjs/winston) [morgan](https://github.com/expressjs/morgan) *custom example available*
 | CI  | [Travis CI](https://travis-ci.org/weareopensource/Node)
 | Developer  | [Coveralls](https://coveralls.io/github/weareopensource/Node) - [Code Climate](https://codeclimate.com/github/weareopensource/Node) - [Dependency status](https://david-dm.org/weareopensource/node) - [Dependabot](https://dependabot.com/) - [Snyk](https://snyk.io/test/github/weareopensource/node) <br> [standard-version](https://github.com/conventional-changelog/standard-version) - [commitlint](https://github.com/conventional-changelog/commitlint) - [commitizen](https://github.com/commitizen/cz-cli) - [waos-conventional-changelog](https://github.com/WeAreOpenSourceProjects/waos-conventional-changelog)
@@ -51,9 +52,10 @@ Our stack node is actually in Beta.
 
 #### Available
 
-* **User** : classic register / auth or oAuth(microsoft, google) - profile management (update, avatar upload ...) - **data privacy** (delete all, get all, send all by mail)
-* **Admin** : list users - edit user - delete user
-* **Tasks** : list tasks - add tasks - edit tasks - delete tasks
+* **User** : classic register / auth or oAuth(microsoft, google) - profile management (update, avatar upload ...) - **data privacy ok** (delete all data, get all data, send all by mail data)
+* **Admin** : list users - get user - edit user - delete user
+* **Tasks** : list tasks - get task - add tasks - edit tasks - delete tasks - **data privacy ok**
+* **Uploads** : get upload - add upload - delete upload  **data privacy ok**
 
 ## Prerequisites
 

config/defaults/development.js

@@ -83,17 +83,17 @@ module.exports = {
   },
   whitelists: {
     users: {
-      default: ['_id', 'id', 'firstName', 'lastName', 'displayName', 'email', 'roles', 'profileImageURL', 'resetPasswordToken', 'resetPasswordExpires'],
-      update: ['firstName', 'lastName', 'email', 'profileImageURL'],
-      updateAdmin: ['firstName', 'lastName', 'email', 'profileImageURL', 'roles'],
+      default: ['_id', 'id', 'firstName', 'lastName', 'displayName', 'email', 'roles', 'avatar', 'resetPasswordToken', 'resetPasswordExpires'],
+      update: ['firstName', 'lastName', 'email', 'avatar'],
+      updateAdmin: ['firstName', 'lastName', 'email', 'avatar', 'roles'],
       recover: ['password', 'resetPasswordToken', 'resetPasswordExpires'],
       roles: ['user', 'admin'],
     },
   },
   uploads: {
-    profile: {
+    users: {
       avatar: {
-        dest: './uploads',
+        formats: ['image/png', 'image/jpeg', 'image/jpg'],
         limits: {
           fileSize: 1 * 1024 * 1024, // Max file size in bytes (1 MB)
         },

lib/app.js

@@ -7,6 +7,8 @@ const nodeHttps = require('https');
 
 const config = require('../config');
 const mongooseService = require('./services/mongoose');
+const multerService = require('./services/multer');
+
 const express = require('./services/express');
 
 // Establish an SQL server connection, instantiating all models and schemas
@@ -33,6 +35,7 @@ const startMongoose = async () => {
   try {
     await mongooseService.loadModels();
     const dbConnection = await mongooseService.connect();
+    await multerService.setStorage(dbConnection);
     return dbConnection;
   } catch (e) {
     throw new Error(e);

lib/helpers/errors.js

@@ -71,12 +71,16 @@ const getMessageFromCode = (err) => {
 const getMessageFromErrors = (err) => {
   if (err instanceof Array) return err.errors;
   let output = '';
-  err.errors.map((error) => {
-    if (error.message) {
-      output = error.message;
-    }
-    return null;
-  });
+  if (err.errors instanceof Array) {
+    err.errors.map((error) => {
+      if (error.message) {
+        output = error.message;
+      }
+      return null;
+    });
+  } else {
+    output = err.message;
+  }
   return output;
 };
 

lib/services/mongoose.js

@@ -15,7 +15,6 @@ module.exports.loadModels = (callback) => {
   config.files.mongooseModels.forEach((modelPath) => {
     require(path.resolve(modelPath));
   });
-
   if (callback) callback();
 };
 
@@ -29,7 +28,7 @@ module.exports.connect = () => new Promise((resolve, reject) => {
   // see: http://mongoosejs.com/docs/connections.html#use-mongo-client
   const mongoOptions = config.db.options;
 
-  mongoose
+  return mongoose
     .connect(config.db.uri, mongoOptions)
     .then(() => {
       // Enabling mongoose debug mode if required

lib/services/multer.js

@@ -1,8 +1,70 @@
-module.exports.imageFileFilter = (req, file, callback) => {
-  if (file.mimetype !== 'image/png' && file.mimetype !== 'image/jpg' && file.mimetype !== 'image/jpeg' && file.mimetype !== 'image/gif') {
-    const err = new Error();
-    err.code = 'UNSUPPORTED_MEDIA_TYPE';
-    return callback(err, false);
+/**
+ * Module dependencies.
+ */
+const path = require('path');
+const crypto = require('crypto');
+const multer = require('multer');
+const { createBucket } = require('mongoose-gridfs');
+
+const AppError = require(path.resolve('./lib/helpers/AppError'));
+
+let storage;
+
+/**
+ * @desc File filter
+ * @param {Array} formats - array of accepted mimetype string
+ * @return {callback}
+ */
+const fileFilter = (formats) => (req, file, callback) => {
+  if (formats.includes(file.mimetype)) callback(null, true);
+  else callback(new AppError(`Only ${formats} images allowed`, { code: 'SERVICE_ERROR' }), false);
+};
+
+/**
+ * set Strorage
+ */
+module.exports.setStorage = () => {
+  storage = createBucket({
+    bucketName: 'uploads',
+    model: 'Uploads',
+  });
+};
+
+/**
+ * @desc file upload middleware
+ * @param {String} name - key data name in form-data
+ * @param {Object} config - multer config
+ * @return {callback}
+ */
+module.exports.create = (name, config) => async (req, res, next) => {
+  // set options
+  const options = config || {};
+  if (options.formats) {
+    options.fileFilter = fileFilter(options.formats);
+    delete options.formats;
   }
-  callback(null, true);
+  // set storage
+  options.storage = storage;
+  // upload
+  const upload = multer(options).single(name);
+  upload(req, res, (err) => {
+    if (err) {
+      req.multerErr = err;
+      next();
+    } else {
+      next();
+    }
+  });
 };
+
+/**
+ * @desc Generate file name
+ * @param {String} filename - original filename
+ * @return {resolve}
+ */
+module.exports.generateFileName = (filename) => new Promise((resolve, reject) => {
+  crypto.randomBytes(32, (err, buf) => {
+    if (err) reject(new AppError('Error generateFileName', { code: 'SERVICE_ERROR' }));
+    resolve(buf.toString('hex') + path.extname(filename));
+  });
+});

modules/uploads/controllers/uploads.controller.js

@@ -0,0 +1,62 @@
+/**
+ * Module dependencies
+ */
+const path = require('path');
+
+const errors = require(path.resolve('./lib/helpers/errors'));
+const responses = require(path.resolve('./lib/helpers/responses'));
+const UploadsService = require('../services/uploads.service');
+
+/**
+ * @desc Endpoint to get an upload by fileName
+ * @param {Object} req - Express request object
+ * @param {Object} res - Express response object
+ */
+exports.get = async (req, res) => {
+  try {
+    const stream = await UploadsService.getStream({ _id: req.upload._id });
+    if (!stream) responses.error(res, 404, 'Not Found', 'No Upload with that identifier can been found')();
+    stream.on('error', (err) => {
+      responses.error(res, 422, 'Unprocessable Entity', errors.getMessage(err))(err);
+    });
+    res.set('Content-Type', req.upload.contentType);
+    stream.pipe(res);
+  } catch (err) {
+    responses.error(res, 422, 'Unprocessable Entity', errors.getMessage(err))(err);
+  }
+};
+
+/**
+ * @desc Endpoint to delete an upload
+ * @param {Object} req - Express request object
+ * @param {Object} res - Express response object
+ */
+exports.delete = async (req, res) => {
+  try {
+    await UploadsService.delete({ _id: req.upload._id });
+    responses.success(res, 'upload deleted')();
+  } catch (err) {
+    responses.error(res, 422, 'Unprocessable Entity', errors.getMessage(err))(err);
+  }
+};
+
+/**
+ * @desc MiddleWare to ask the service the uppload for this uploadName
+ * @param {Object} req - Express request object
+ * @param {Object} res - Express response object
+ * @param {Function} next - Express next middleware function
+ * @param {String} filename - upload filename
+ */
+exports.uploadByName = async (req, res, next, uploadName) => {
+  try {
+    const upload = await UploadsService.get(uploadName);
+    if (!upload) responses.error(res, 404, 'Not Found', 'No Upload with that name has been found')();
+    else {
+      req.upload = upload;
+      req.isOwner = upload.metadata.user; // used if we proteck road by isOwner policy
+      next();
+    }
+  } catch (err) {
+    next(err);
+  }
+};

modules/uploads/models/uploads.model.mongoose.js

@@ -0,0 +1,26 @@
+/**
+ * Module dependencies
+ */
+const mongoose = require('mongoose');
+
+const Schema = mongoose.Schema;
+
+/**
+ * Data Model Mongoose
+ */
+const UploadsMongoose = new Schema({
+  length: Number,
+  chunkSize: Number,
+  uploadDate: Date,
+  md5: String,
+  filename: String,
+  contentType: String,
+  metadata: {
+    user: {
+      type: Schema.ObjectId,
+      ref: 'User',
+    },
+  },
+}, { strict: false });
+
+mongoose.model('Uploads', UploadsMongoose, 'uploads.files');

modules/uploads/policies/uploads.policy.js

@@ -0,0 +1,19 @@
+/**
+ * Module dependencies
+* */
+const path = require('path');
+
+const policy = require(path.resolve('./lib/middlewares/policy'));
+
+/**
+ * Invoke Uploads Permissions
+ */
+exports.invokeRolesPolicies = () => {
+  policy.Acl.allow([{
+    roles: ['user', 'admin'],
+    allows: [{
+      resources: '/api/uploads/:uploadName',
+      permissions: ['get', 'delete'],
+    }],
+  }]);
+};

modules/uploads/repositories/uploads.repository.js

@@ -0,0 +1,70 @@
+/**
+ * Module dependencies
+ */
+const mongoose = require('mongoose');
+const { createModel } = require('mongoose-gridfs');
+const path = require('path');
+
+const AppError = require(path.resolve('./lib/helpers/AppError'));
+const Attachment = createModel({ bucketName: 'uploads' });
+const Uploads = mongoose.model('Uploads');
+
+
+/**
+ * @desc Function to get all upload in db with filter or not
+ * @return {Array} uploads
+ */
+exports.list = (filter) => Uploads.find(filter).select('filename uploadDate contentType').sort('-createdAt').exec();
+
+/**
+ * @desc Function to get an upload from db
+ * @param {String} id
+ * @return {Stream} upload
+ */
+exports.get = (uploadName) => Uploads.findOne({ filename: uploadName }).exec();
+
+/**
+ * @desc Function to get an upload from db
+ * @param {String} id
+ * @return {Stream} upload
+ */
+exports.getStream = (upload) => Attachment.read(upload);
+
+/**
+ * @desc Function to update an upload in db
+ * @param {ObjectID} upload ID
+ * @param {Object} update
+ * @return {Object} upload updated
+ */
+exports.update = (id, update) => Uploads.findOneAndUpdate({ _id: id }, update, { new: true }).exec();
+
+/**
+ * @desc Function to delete an upload from db
+ * @param {String} id
+ * @return {Object} confirmation of delete
+ */
+exports.delete = async (upload) => {
+  if (!upload._id) upload = await Uploads.findOne({ filename: upload.filename }).exec();
+  if (upload) {
+    Attachment.unlink(upload._id, (err, unlinked) => {
+      if (err) throw new AppError('Upload: delete error', { code: 'REPOSITORY_ERROR', details: err });
+      return unlinked;
+    });
+  }
+};
+
+/**
+ * @desc Function to delete uploads of one user in db
+ * @param {Object} filter
+ * @return {Object} confirmation of delete
+ */
+exports.deleteMany = async (filter) => {
+  const uploads = await this.list(filter);
+  uploads.forEach((upload) => {
+    Attachment.unlink(upload._id, (err, unlinked) => {
+      if (err) throw new AppError('Upload: delete error', { code: 'REPOSITORY_ERROR', details: err });
+      return unlinked;
+    });
+  });
+  return { deletedCount: uploads.length };
+};

modules/uploads/routes/uploads.routes.js

@@ -0,0 +1,21 @@
+/**
+ * Module dependencies
+ */
+const passport = require('passport');
+const path = require('path');
+
+const policy = require(path.resolve('./lib/middlewares/policy'));
+const uploads = require('../controllers/uploads.controller');
+
+/**
+ * Routes
+ */
+module.exports = (app) => {
+  // classic crud
+  app.route('/api/uploads/:uploadName').all(passport.authenticate('jwt'), policy.isAllowed)
+    .get(uploads.get)
+    .delete(policy.isOwner, uploads.delete); // delete
+
+  // Finish by binding the task middleware
+  app.param('uploadName', uploads.uploadByName);
+};