Companion server upload events

packages/@uppy/companion/src/companion.js

@@ -1,51 +1,53 @@
-const fs = require('fs')
 const express = require('express')
-const ms = require('ms')
 // @ts-ignore
 const Grant = require('grant').express()
 const merge = require('lodash.merge')
 const cookieParser = require('cookie-parser')
 const interceptor = require('express-interceptor')
-const { isURL } = require('validator')
 const uuid = require('uuid')
 
 const grantConfig = require('./config/grant')()
 const providerManager = require('./server/provider')
 const controllers = require('./server/controllers')
 const s3 = require('./server/controllers/s3')
-const getS3Client = require('./server/s3-client')
 const url = require('./server/controllers/url')
-const emitter = require('./server/emitter')
+const createEmitter = require('./server/emitter')
 const redis = require('./server/redis')
 const { getURLBuilder } = require('./server/helpers/utils')
 const jobs = require('./server/jobs')
 const logger = require('./server/logger')
 const middlewares = require('./server/middlewares')
+const { getMaskableSecrets, defaultOptions, validateConfig } = require('./config/companion')
 const { ProviderApiError, ProviderAuthError } = require('./server/provider/error')
 const { getCredentialsOverrideMiddleware } = require('./server/provider/credentials')
 // @ts-ignore
 const { version } = require('../package.json')
 
-const defaultOptions = {
-  server: {
-    protocol: 'http',
-    path: '',
-  },
-  providerOptions: {
-    s3: {
-      acl: 'public-read',
-      endpoint: 'https://{service}.{region}.amazonaws.com',
-      conditions: [],
-      useAccelerateEndpoint: false,
-      getKey: (req, filename) => filename,
-      expires: ms('5 minutes') / 1000,
+// intercepts grantJS' default response error when something goes
+// wrong during oauth process.
+const interceptGrantErrorResponse = interceptor((req, res) => {
+  return {
+    isInterceptable: () => {
+      // match grant.js' callback url
+      return /^\/connect\/\w+\/callback/.test(req.path)
     },
-  },
-  allowLocalUrls: false,
-  logClientVersion: true,
-  periodicPingUrls: [],
-  streamingUpload: false,
-}
+    intercept: (body, send) => {
+      const unwantedBody = 'error=Grant%3A%20missing%20session%20or%20misconfigured%20provider'
+      if (body === unwantedBody) {
+        logger.error(`grant.js responded with error: ${body}`, 'grant.oauth.error', req.id)
+        res.set('Content-Type', 'text/plain')
+        const reqHint = req.id ? `Request ID: ${req.id}` : ''
+        send([
+          'Companion was unable to complete the OAuth process :(',
+          'Error: User session is missing or the Provider was misconfigured',
+          reqHint,
+        ].join('\n'))
+      } else {
+        send(body)
+      }
+    },
+  }
+})
 
 // make the errors available publicly for custom providers
 module.exports.errors = { ProviderApiError, ProviderAuthError }
@@ -54,13 +56,13 @@ module.exports.socket = require('./server/socket')
 /**
  * Entry point into initializing the Companion app.
  *
- * @param {object} options
+ * @param {object} optionsArg
  * @returns {import('express').Express}
  */
-module.exports.app = (options = {}) => {
-  validateConfig(options)
+module.exports.app = (optionsArg = {}) => {
+  validateConfig(optionsArg)
 
-  options = merge({}, defaultOptions, options)
+  const options = merge({}, defaultOptions, optionsArg)
   const providers = providerManager.getDefaultProviders()
   const searchProviders = providerManager.getSearchProviders()
   providerManager.addProviderOptions(options, grantConfig)
@@ -71,13 +73,13 @@ module.exports.app = (options = {}) => {
   }
 
   // mask provider secrets from log messages
-  maskLogger(options)
+  logger.setMaskables(getMaskableSecrets(options))
 
   // create singleton redis client
   if (options.redisUrl) {
     redis.client(merge({ url: options.redisUrl }, options.redisOptions || {}))
   }
-  emitter(options.multipleInstances && options.redisUrl, options.redisPubSubScope)
+  const emitter = createEmitter(options.multipleInstances && options.redisUrl, options.redisPubSubScope)
 
   const app = express()
 
@@ -114,7 +116,7 @@ module.exports.app = (options = {}) => {
   app.use(middlewares.cors(options))
 
   // add uppy options to the request object so it can be accessed by subsequent handlers.
-  app.use('*', getOptionsMiddleware(options))
+  app.use('*', middlewares.getCompanionMiddleware(options))
   app.use('/s3', s3(options.providerOptions.s3))
   app.use('/url', url())
 
@@ -150,143 +152,8 @@ module.exports.app = (options = {}) => {
     processId,
   })
 
+  // todo split emitter from app in next major
+  // @ts-ignore
+  app.companionEmitter = emitter
   return app
 }
-
-// intercepts grantJS' default response error when something goes
-// wrong during oauth process.
-const interceptGrantErrorResponse = interceptor((req, res) => {
-  return {
-    isInterceptable: () => {
-      // match grant.js' callback url
-      return /^\/connect\/\w+\/callback/.test(req.path)
-    },
-    intercept: (body, send) => {
-      const unwantedBody = 'error=Grant%3A%20missing%20session%20or%20misconfigured%20provider'
-      if (body === unwantedBody) {
-        logger.error(`grant.js responded with error: ${body}`, 'grant.oauth.error', req.id)
-        res.set('Content-Type', 'text/plain')
-        const reqHint = req.id ? `Request ID: ${req.id}` : ''
-        send([
-          'Companion was unable to complete the OAuth process :(',
-          'Error: User session is missing or the Provider was misconfigured',
-          reqHint,
-        ].join('\n'))
-      } else {
-        send(body)
-      }
-    },
-  }
-})
-
-/**
- *
- * @param {object} options
- */
-const getOptionsMiddleware = (options) => {
-  /**
-   * @param {object} req
-   * @param {object} res
-   * @param {Function} next
-   */
-  const middleware = (req, res, next) => {
-    const versionFromQuery = req.query.uppyVersions ? decodeURIComponent(req.query.uppyVersions) : null
-    req.companion = {
-      options,
-      s3Client: getS3Client(options),
-      authToken: req.header('uppy-auth-token') || req.query.uppyAuthToken,
-      clientVersion: req.header('uppy-versions') || versionFromQuery || '1.0.0',
-      buildURL: getURLBuilder(options),
-    }
-
-    if (options.logClientVersion) {
-      logger.info(`uppy client version ${req.companion.clientVersion}`, 'companion.client.version')
-    }
-    next()
-  }
-
-  return middleware
-}
-
-/**
- * Informs the logger about all provider secrets that should be masked
- * if they are found in a log message
- *
- * @param {object} companionOptions
- */
-const maskLogger = (companionOptions) => {
-  const secrets = []
-  const { providerOptions, customProviders } = companionOptions
-  Object.keys(providerOptions).forEach((provider) => {
-    if (providerOptions[provider].secret) {
-      secrets.push(providerOptions[provider].secret)
-    }
-  })
-
-  if (customProviders) {
-    Object.keys(customProviders).forEach((provider) => {
-      if (customProviders[provider].config && customProviders[provider].config.secret) {
-        secrets.push(customProviders[provider].config.secret)
-      }
-    })
-  }
-
-  logger.setMaskables(secrets)
-}
-
-/**
- * validates that the mandatory companion options are set.
- * If it is invalid, it will console an error of unset options and exits the process.
- * If it is valid, nothing happens.
- *
- * @param {object} companionOptions
- */
-const validateConfig = (companionOptions) => {
-  const mandatoryOptions = ['secret', 'filePath', 'server.host']
-  /** @type {string[]} */
-  const unspecified = []
-
-  mandatoryOptions.forEach((i) => {
-    const value = i.split('.').reduce((prev, curr) => (prev ? prev[curr] : undefined), companionOptions)
-
-    if (!value) unspecified.push(`"${i}"`)
-  })
-
-  // vaidate that all required config is specified
-  if (unspecified.length) {
-    const messagePrefix = 'Please specify the following options to use companion:'
-    throw new Error(`${messagePrefix}\n${unspecified.join(',\n')}`)
-  }
-
-  // validate that specified filePath is writeable/readable.
-  try {
-    // @ts-ignore
-    fs.accessSync(`${companionOptions.filePath}`, fs.R_OK | fs.W_OK) // eslint-disable-line no-bitwise
-  } catch (err) {
-    throw new Error(
-      `No access to "${companionOptions.filePath}". Please ensure the directory exists and with read/write permissions.`,
-    )
-  }
-
-  const { providerOptions, periodicPingUrls } = companionOptions
-
-  if (providerOptions) {
-    const deprecatedOptions = { microsoft: 'onedrive', google: 'drive' }
-    Object.keys(deprecatedOptions).forEach((deprected) => {
-      if (providerOptions[deprected]) {
-        throw new Error(`The Provider option "${deprected}" is no longer supported. Please use the option "${deprecatedOptions[deprected]}" instead.`)
-      }
-    })
-  }
-
-  if (companionOptions.uploadUrls == null || companionOptions.uploadUrls.length === 0) {
-    logger.warn('Running without uploadUrls specified is a security risk if running in production', 'startup.uploadUrls')
-  }
-
-  if (periodicPingUrls != null && (
-    !Array.isArray(periodicPingUrls)
-    || periodicPingUrls.some((url2) => !isURL(url2, { protocols: ['http', 'https'], require_protocol: true, require_tld: false }))
-  )) {
-    throw new TypeError('Invalid periodicPingUrls')
-  }
-}

packages/@uppy/companion/src/config/companion.js

@@ -0,0 +1,112 @@
+const ms = require('ms')
+const fs = require('fs')
+const { isURL } = require('validator')
+const logger = require('../server/logger')
+
+const defaultOptions = {
+  server: {
+    protocol: 'http',
+    path: '',
+  },
+  providerOptions: {
+    s3: {
+      acl: 'public-read',
+      endpoint: 'https://{service}.{region}.amazonaws.com',
+      conditions: [],
+      useAccelerateEndpoint: false,
+      getKey: (req, filename) => filename,
+      expires: ms('5 minutes') / 1000,
+    },
+  },
+  allowLocalUrls: false,
+  logClientVersion: true,
+  periodicPingUrls: [],
+  streamingUpload: false,
+  clientSocketConnectTimeout: 60000,
+}
+
+/**
+ * @param {object} companionOptions
+ */
+function getMaskableSecrets (companionOptions) {
+  const secrets = []
+  const { providerOptions, customProviders } = companionOptions
+  Object.keys(providerOptions).forEach((provider) => {
+    if (providerOptions[provider].secret) {
+      secrets.push(providerOptions[provider].secret)
+    }
+  })
+
+  if (customProviders) {
+    Object.keys(customProviders).forEach((provider) => {
+      if (customProviders[provider].config && customProviders[provider].config.secret) {
+        secrets.push(customProviders[provider].config.secret)
+      }
+    })
+  }
+
+  return secrets
+}
+
+/**
+ * validates that the mandatory companion options are set.
+ * If it is invalid, it will console an error of unset options and exits the process.
+ * If it is valid, nothing happens.
+ *
+ * @param {object} companionOptions
+ */
+const validateConfig = (companionOptions) => {
+  const mandatoryOptions = ['secret', 'filePath', 'server.host']
+  /** @type {string[]} */
+  const unspecified = []
+
+  mandatoryOptions.forEach((i) => {
+    const value = i.split('.').reduce((prev, curr) => (prev ? prev[curr] : undefined), companionOptions)
+
+    if (!value) unspecified.push(`"${i}"`)
+  })
+
+  // vaidate that all required config is specified
+  if (unspecified.length) {
+    const messagePrefix = 'Please specify the following options to use companion:'
+    throw new Error(`${messagePrefix}\n${unspecified.join(',\n')}`)
+  }
+
+  // validate that specified filePath is writeable/readable.
+  try {
+    // @ts-ignore
+    fs.accessSync(`${companionOptions.filePath}`, fs.R_OK | fs.W_OK) // eslint-disable-line no-bitwise
+  } catch (err) {
+    throw new Error(
+      `No access to "${companionOptions.filePath}". Please ensure the directory exists and with read/write permissions.`,
+    )
+  }
+
+  const { providerOptions, periodicPingUrls } = companionOptions
+
+  if (providerOptions) {
+    const deprecatedOptions = { microsoft: 'onedrive', google: 'drive' }
+    Object.keys(deprecatedOptions).forEach((deprected) => {
+      if (providerOptions[deprected]) {
+        throw new Error(`The Provider option "${deprected}" is no longer supported. Please use the option "${deprecatedOptions[deprected]}" instead.`)
+      }
+    })
+  }
+
+  if (companionOptions.uploadUrls == null || companionOptions.uploadUrls.length === 0) {
+    logger.warn('Running without uploadUrls specified is a security risk if running in production', 'startup.uploadUrls')
+  }
+
+  if (periodicPingUrls != null && (
+    !Array.isArray(periodicPingUrls)
+    || periodicPingUrls.some((url2) => !isURL(url2, { protocols: ['http', 'https'], require_protocol: true, require_tld: false }))
+  )) {
+    throw new TypeError('Invalid periodicPingUrls')
+  }
+}
+
+module.exports = {
+  defaultOptions,
+  getMaskableSecrets,
+  validateConfig,
+}