feat: modular auth + queue tasks

config.sample.yml

@@ -5,7 +5,7 @@
 # https://docs.requarks.io/wiki/install
 
 # ---------------------------------------------------------------------
-# Port the main server should listen to (80 by default)
+# Port the main server should listen to
 # ---------------------------------------------------------------------
 
 port: 80
@@ -39,3 +39,9 @@ redis:
   db: 0
   password: null
 
+# ---------------------------------------------------------------------
+# Background Workers
+# ---------------------------------------------------------------------
+
+# Leave 0 for auto based on CPU cores
+workers: 0

package.json

@@ -52,7 +52,6 @@
     "connect-flash": "~0.1.1",
     "connect-redis": "~3.3.0",
     "cookie-parser": "~1.4.3",
-    "cron": "~1.2.1",
     "diff2html": "~2.3.0",
     "execa": "~0.7.0",
     "express": "~4.15.3",

server/app/data.yml

@@ -5,65 +5,22 @@
 name: Wiki.js
 defaults:
   config:
-    title: Wiki
-    host: http://localhost
     port: 80
     paths:
       repo: ./repo
       data: ./data
-    uploads:
-      maxImageFileSize: 3,
-      maxOtherFileSize: 100
-    lang: en
-    public: false
-    auth:
-      defaultReadAccess: false
-      local:
-        enabled: true
-      microsoft:
-        enabled: false
-      google:
-        enabled: false
-      facebook:
-        enabled: false
-      github:
-        enabled: false
-      slack:
-        enabled: false
-      ldap:
-        enabled: false
-      azure:
-        enabled: false
-    db: mongodb://localhost/wiki
-    sessionSecret: null
-    admin: null
-    git:
-      url: null
-      branch: master
-      auth:
-        type: basic
-        username: null
-        password: null
-        privateKey: null
-        sslVerify: true
-      serverEmail: wiki@example.com
-      showUserEmail: true
-    features:
-      linebreaks: true
-      mathjax: true
-    externalLogging:
-      bugsnap: false
-      loggly: false
-      papertrail: false
-      rollbar: false
-      sentry: false
-    theme:
-      primary: indigo
-      alt: blue-grey
-      footer: blue-grey
-      code:
-        dark: true
-        colorize: true
+    db:
+      host: localhost
+      port: 5432
+      user: wikijs
+      pass: wikijsrocks
+      db: wiki
+    redis:
+      host: localhost
+      port: 6379
+      db: 0
+      password: null
+    workers: 0
 authProviders:
   - local
   - microsoft
@@ -113,6 +70,9 @@ langs:
     id: ko
     name: Korean - 한국어
   -
+    id: pt
+    name: Portuguese - Português
+  -
     id: ru
     name: Russian - Русский
   -

server/authentication/azure.js

+'use strict'
+
+/* global wiki */
+
+// ------------------------------------
+// Azure AD Account
+// ------------------------------------
+
+const AzureAdOAuth2Strategy = require('passport-azure-ad-oauth2').Strategy
+
+module.exports = (passport) => {
+  if (wiki.config.auth.azure && wiki.config.auth.azure.enabled) {
+    const jwt = require('jsonwebtoken')
+    passport.use('azure_ad_oauth2',
+      new AzureAdOAuth2Strategy({
+        clientID: wiki.config.auth.azure.clientId,
+        clientSecret: wiki.config.auth.azure.clientSecret,
+        callbackURL: wiki.config.host + '/login/azure/callback',
+        resource: wiki.config.auth.azure.resource,
+        tenant: wiki.config.auth.azure.tenant
+      }, (accessToken, refreshToken, params, profile, cb) => {
+        let waadProfile = jwt.decode(params.id_token)
+        waadProfile.id = waadProfile.oid
+        waadProfile.provider = 'azure'
+        wiki.db.User.processProfile(waadProfile).then((user) => {
+          return cb(null, user) || true
+        }).catch((err) => {
+          return cb(err, null) || true
+        })
+      }
+      ))
+  }
+}

server/authentication/facebook.js

+'use strict'
+
+/* global wiki */
+
+// ------------------------------------
+// Facebook Account
+// ------------------------------------
+
+const FacebookStrategy = require('passport-facebook').Strategy
+
+module.exports = (passport) => {
+  if (wiki.config.auth.facebook && wiki.config.auth.facebook.enabled) {
+    passport.use('facebook',
+      new FacebookStrategy({
+        clientID: wiki.config.auth.facebook.clientId,
+        clientSecret: wiki.config.auth.facebook.clientSecret,
+        callbackURL: wiki.config.host + '/login/facebook/callback',
+        profileFields: ['id', 'displayName', 'email']
+      }, function (accessToken, refreshToken, profile, cb) {
+        wiki.db.User.processProfile(profile).then((user) => {
+          return cb(null, user) || true
+        }).catch((err) => {
+          return cb(err, null) || true
+        })
+      }
+      ))
+  }
+}

server/authentication/github.js

+'use strict'
+
+/* global wiki */
+
+// ------------------------------------
+// GitHub Account
+// ------------------------------------
+
+const GitHubStrategy = require('passport-github2').Strategy
+
+module.exports = (passport) => {
+  if (wiki.config.auth.github && wiki.config.auth.github.enabled) {
+    passport.use('github',
+      new GitHubStrategy({
+        clientID: wiki.config.auth.github.clientId,
+        clientSecret: wiki.config.auth.github.clientSecret,
+        callbackURL: wiki.config.host + '/login/github/callback',
+        scope: ['user:email']
+      }, (accessToken, refreshToken, profile, cb) => {
+        wiki.db.User.processProfile(profile).then((user) => {
+          return cb(null, user) || true
+        }).catch((err) => {
+          return cb(err, null) || true
+        })
+      }
+      ))
+  }
+}

server/authentication/google.js

+'use strict'
+
+/* global wiki */
+
+// ------------------------------------
+// Google ID Account
+// ------------------------------------
+
+const GoogleStrategy = require('passport-google-oauth20').Strategy
+
+module.exports = (passport) => {
+  if (wiki.config.auth.google && wiki.config.auth.google.enabled) {
+    passport.use('google',
+      new GoogleStrategy({
+        clientID: wiki.config.auth.google.clientId,
+        clientSecret: wiki.config.auth.google.clientSecret,
+        callbackURL: wiki.config.host + '/login/google/callback'
+      }, (accessToken, refreshToken, profile, cb) => {
+        wiki.db.User.processProfile(profile).then((user) => {
+          return cb(null, user) || true
+        }).catch((err) => {
+          return cb(err, null) || true
+        })
+      }
+      ))
+  }
+}

server/authentication/ldap.js

+'use strict'
+
+/* global wiki */
+
+// ------------------------------------
+// LDAP Account
+// ------------------------------------
+
+const LdapStrategy = require('passport-ldapauth').Strategy
+
+module.exports = (passport) => {
+  if (wiki.config.auth.ldap && wiki.config.auth.ldap.enabled) {
+    passport.use('ldapauth',
+      new LdapStrategy({
+        server: {
+          url: wiki.config.auth.ldap.url,
+          bindDn: wiki.config.auth.ldap.bindDn,
+          bindCredentials: wiki.config.auth.ldap.bindCredentials,
+          searchBase: wiki.config.auth.ldap.searchBase,
+          searchFilter: wiki.config.auth.ldap.searchFilter,
+          searchAttributes: ['displayName', 'name', 'cn', 'mail'],
+          tlsOptions: (wiki.config.auth.ldap.tlsEnabled) ? {
+            ca: [
+              fs.readFileSync(wiki.config.auth.ldap.tlsCertPath)
+            ]
+          } : {}
+        },
+        usernameField: 'email',
+        passReqToCallback: false
+      }, (profile, cb) => {
+        profile.provider = 'ldap'
+        profile.id = profile.dn
+        wiki.db.User.processProfile(profile).then((user) => {
+          return cb(null, user) || true
+        }).catch((err) => {
+          return cb(err, null) || true
+        })
+      }
+      ))
+  }
+}

server/authentication/local.js

+'use strict'
+
+/* global wiki */
+
+// ------------------------------------
+// Local Account
+// ------------------------------------
+
+const LocalStrategy = require('passport-local').Strategy
+
+module.exports = (passport) => {
+  if (wiki.config.auth.local && wiki.config.auth.local.enabled) {
+    passport.use('local',
+      new LocalStrategy({
+        usernameField: 'email',
+        passwordField: 'password'
+      }, (uEmail, uPassword, done) => {
+        wiki.db.User.findOne({ email: uEmail, provider: 'local' }).then((user) => {
+          if (user) {
+            return user.validatePassword(uPassword).then(() => {
+              return done(null, user) || true
+            }).catch((err) => {
+              return done(err, null)
+            })
+          } else {
+            return done(new Error('INVALID_LOGIN'), null)
+          }
+        }).catch((err) => {
+          done(err, null)
+        })
+      }
+      ))
+  }
+}

server/authentication/microsoft.js

+'use strict'
+
+/* global wiki */
+
+// ------------------------------------
+// Microsoft Account
+// ------------------------------------
+
+const WindowsLiveStrategy = require('passport-windowslive').Strategy
+
+module.exports = (passport) => {
+  if (wiki.config.auth.microsoft && wiki.config.auth.microsoft.enabled) {
+    passport.use('windowslive',
+      new WindowsLiveStrategy({
+        clientID: wiki.config.auth.microsoft.clientId,
+        clientSecret: wiki.config.auth.microsoft.clientSecret,
+        callbackURL: wiki.config.host + '/login/ms/callback'
+      }, function (accessToken, refreshToken, profile, cb) {
+        wiki.db.User.processProfile(profile).then((user) => {
+          return cb(null, user) || true
+        }).catch((err) => {
+          return cb(err, null) || true
+        })
+      }
+      ))
+  }
+}

server/authentication/slack.js

+'use strict'
+
+/* global wiki */
+
+// ------------------------------------
+// Slack Account
+// ------------------------------------
+
+const SlackStrategy = require('passport-slack').Strategy
+
+module.exports = (passport) => {
+  if (wiki.config.auth.slack && wiki.config.auth.slack.enabled) {
+    passport.use('slack',
+      new SlackStrategy({
+        clientID: wiki.config.auth.slack.clientId,
+        clientSecret: wiki.config.auth.slack.clientSecret,
+        callbackURL: wiki.config.host + '/login/slack/callback'
+      }, (accessToken, refreshToken, profile, cb) => {
+        wiki.db.User.processProfile(profile).then((user) => {
+          return cb(null, user) || true
+        }).catch((err) => {
+          return cb(err, null) || true
+        })
+      }
+      ))
+  }
+}

server/index.js

@@ -6,10 +6,14 @@
 // ===========================================
 
 const path = require('path')
+const cluster = require('cluster')
+
 let wiki = {
   IS_DEBUG: process.env.NODE_ENV === 'development',
+  IS_MASTER: cluster.isMaster,
   ROOTPATH: process.cwd(),
-  SERVERPATH: path.join(process.cwd(), 'server')
+  SERVERPATH: path.join(process.cwd(), 'server'),
+  configSvc: require('./modules/config')
 }
 global.wiki = wiki
 
@@ -19,29 +23,36 @@ process.env.VIPS_WARNING = false
 //   require('@glimpse/glimpse').init()
 // }
 
-let appconf = require('./modules/config')()
-wiki.config = appconf.config
-wiki.data = appconf.data
+wiki.configSvc.init()
+
+// ----------------------------------------
+// Init Logger
+// ----------------------------------------
+
+wiki.logger = require('./modules/logger').init()
 
 // ----------------------------------------
-// Load Winston
+// Init DB
 // ----------------------------------------
 
-wiki.logger = require('./modules/logger')()
+wiki.db = require('./modules/db').init()
 
 // ----------------------------------------
 // Start Cluster
 // ----------------------------------------
 
-const cluster = require('cluster')
 const numCPUs = require('os').cpus().length
+let numWorkers = (wiki.config.workers > 0) ? wiki.config.workers : numCPUs
+if (numWorkers > numCPUs) {
+  numWorkers = numCPUs
+}
 
 if (cluster.isMaster) {
   wiki.logger.info('Wiki.js is initializing...')
 
   require('./master')
 
-  for (let i = 0; i < numCPUs; i++) {
+  for (let i = 0; i < numWorkers; i++) {
     cluster.fork()
   }
 
@@ -50,5 +61,5 @@ if (cluster.isMaster) {
   })
 } else {
   wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`)
-  // require('./worker')
+  require('./worker')
 }

server/modules/auth.js

@@ -24,232 +24,51 @@ module.exports = function (passport) {
     })
   })
 
-  // Local Account
-
-  if (wiki.config.auth.local && wiki.config.auth.local.enabled) {
-    const LocalStrategy = require('passport-local').Strategy
-    passport.use('local',
-      new LocalStrategy({
-        usernameField: 'email',
-        passwordField: 'password'
-      }, (uEmail, uPassword, done) => {
-        wiki.db.User.findOne({ email: uEmail, provider: 'local' }).then((user) => {
-          if (user) {
-            return user.validatePassword(uPassword).then(() => {
-              return done(null, user) || true
-            }).catch((err) => {
-              return done(err, null)
-            })
-          } else {
-            return done(new Error('INVALID_LOGIN'), null)
-          }
-        }).catch((err) => {
-          done(err, null)
-        })
-      }
-      ))
-  }
-
-  // Google ID
-
-  if (wiki.config.auth.google && wiki.config.auth.google.enabled) {
-    const GoogleStrategy = require('passport-google-oauth20').Strategy
-    passport.use('google',
-      new GoogleStrategy({
-        clientID: wiki.config.auth.google.clientId,
-        clientSecret: wiki.config.auth.google.clientSecret,
-        callbackURL: wiki.config.host + '/login/google/callback'
-      }, (accessToken, refreshToken, profile, cb) => {
-        wiki.db.User.processProfile(profile).then((user) => {
-          return cb(null, user) || true
-        }).catch((err) => {
-          return cb(err, null) || true
-        })
-      }
-      ))
-  }
-
-  // Microsoft Accounts
-
-  if (wiki.config.auth.microsoft && wiki.config.auth.microsoft.enabled) {
-    const WindowsLiveStrategy = require('passport-windowslive').Strategy
-    passport.use('windowslive',
-      new WindowsLiveStrategy({
-        clientID: wiki.config.auth.microsoft.clientId,
-        clientSecret: wiki.config.auth.microsoft.clientSecret,
-        callbackURL: wiki.config.host + '/login/ms/callback'
-      }, function (accessToken, refreshToken, profile, cb) {
-        wiki.db.User.processProfile(profile).then((user) => {
-          return cb(null, user) || true
-        }).catch((err) => {
-          return cb(err, null) || true
-        })
-      }
-      ))
-  }
-
-  // Facebook
-
-  if (wiki.config.auth.facebook && wiki.config.auth.facebook.enabled) {
-    const FacebookStrategy = require('passport-facebook').Strategy
-    passport.use('facebook',
-      new FacebookStrategy({
-        clientID: wiki.config.auth.facebook.clientId,
-        clientSecret: wiki.config.auth.facebook.clientSecret,
-        callbackURL: wiki.config.host + '/login/facebook/callback',
-        profileFields: ['id', 'displayName', 'email']
-      }, function (accessToken, refreshToken, profile, cb) {
-        wiki.db.User.processProfile(profile).then((user) => {
-          return cb(null, user) || true
-        }).catch((err) => {
-          return cb(err, null) || true
-        })
-      }
-      ))
-  }
-
-  // GitHub
-
-  if (wiki.config.auth.github && wiki.config.auth.github.enabled) {
-    const GitHubStrategy = require('passport-github2').Strategy
-    passport.use('github',
-      new GitHubStrategy({
-        clientID: wiki.config.auth.github.clientId,
-        clientSecret: wiki.config.auth.github.clientSecret,
-        callbackURL: wiki.config.host + '/login/github/callback',
-        scope: ['user:email']
-      }, (accessToken, refreshToken, profile, cb) => {
-        wiki.db.User.processProfile(profile).then((user) => {
-          return cb(null, user) || true
-        }).catch((err) => {
-          return cb(err, null) || true
-        })
-      }
-      ))
-  }
-
-  // Slack
-
-  if (wiki.config.auth.slack && wiki.config.auth.slack.enabled) {
-    const SlackStrategy = require('passport-slack').Strategy
-    passport.use('slack',
-      new SlackStrategy({
-        clientID: wiki.config.auth.slack.clientId,
-        clientSecret: wiki.config.auth.slack.clientSecret,
-        callbackURL: wiki.config.host + '/login/slack/callback'
-      }, (accessToken, refreshToken, profile, cb) => {
-        wiki.db.User.processProfile(profile).then((user) => {
-          return cb(null, user) || true
-        }).catch((err) => {
-          return cb(err, null) || true
-        })
-      }
-      ))
-  }
-
-  // LDAP
-
-  if (wiki.config.auth.ldap && wiki.config.auth.ldap.enabled) {
-    const LdapStrategy = require('passport-ldapauth').Strategy
-    passport.use('ldapauth',
-      new LdapStrategy({
-        server: {
-          url: wiki.config.auth.ldap.url,
-          bindDn: wiki.config.auth.ldap.bindDn,
-          bindCredentials: wiki.config.auth.ldap.bindCredentials,
-          searchBase: wiki.config.auth.ldap.searchBase,
-          searchFilter: wiki.config.auth.ldap.searchFilter,
-          searchAttributes: ['displayName', 'name', 'cn', 'mail'],
-          tlsOptions: (wiki.config.auth.ldap.tlsEnabled) ? {
-            ca: [
-              fs.readFileSync(wiki.config.auth.ldap.tlsCertPath)
-            ]
-          } : {}
-        },
-        usernameField: 'email',
-        passReqToCallback: false
-      }, (profile, cb) => {
-        profile.provider = 'ldap'
-        profile.id = profile.dn
-        wiki.db.User.processProfile(profile).then((user) => {
-          return cb(null, user) || true
-        }).catch((err) => {
-          return cb(err, null) || true
-        })
-      }
-      ))
-  }
-
-  // AZURE AD
-
-  if (wiki.config.auth.azure && wiki.config.auth.azure.enabled) {
-    const AzureAdOAuth2Strategy = require('passport-azure-ad-oauth2').Strategy
-    const jwt = require('jsonwebtoken')
-    passport.use('azure_ad_oauth2',
-      new AzureAdOAuth2Strategy({
-        clientID: wiki.config.auth.azure.clientId,
-        clientSecret: wiki.config.auth.azure.clientSecret,
-        callbackURL: wiki.config.host + '/login/azure/callback',
-        resource: wiki.config.auth.azure.resource,
-        tenant: wiki.config.auth.azure.tenant
-      }, (accessToken, refreshToken, params, profile, cb) => {
-        let waadProfile = jwt.decode(params.id_token)
-        waadProfile.id = waadProfile.oid
-        waadProfile.provider = 'azure'
-        wiki.db.User.processProfile(waadProfile).then((user) => {
-          return cb(null, user) || true
-        }).catch((err) => {
-          return cb(err, null) || true
-        })
-      }
-      ))
-  }
-
   // Create users for first-time
 
-  wiki.db.onReady.then(() => {
-    return wiki.db.User.findOne({ provider: 'local', email: 'guest' }).then((c) => {
-      if (c < 1) {
-        // Create guest account
-
-        return wiki.db.User.create({
-          provider: 'local',
-          email: 'guest@example.com',
-          name: 'Guest',
-          password: '',
-          role: 'guest'
-        }).then(() => {
-          wiki.logger.info('[AUTH] Guest account created successfully!')
-          return true
-        }).catch((err) => {
-          wiki.logger.error('[AUTH] An error occured while creating guest account:')
-          wiki.logger.error(err)
-          return err
-        })
-      }
-    }).then(() => {
-      if (process.env.WIKI_JS_HEROKU) {
-        return wiki.db.User.findOne({ provider: 'local', email: process.env.WIKI_ADMIN_EMAIL }).then((c) => {
-          if (c < 1) {
-            // Create root admin account (HEROKU ONLY)
-
-            return wiki.db.User.create({
-              provider: 'local',
-              email: process.env.WIKI_ADMIN_EMAIL,
-              name: 'Administrator',
-              password: '$2a$04$MAHRw785Xe/Jd5kcKzr3D.VRZDeomFZu2lius4gGpZZ9cJw7B7Mna', // admin123 (default)
-              role: 'admin'
-            }).then(() => {
-              wiki.logger.info('[AUTH] Root admin account created successfully!')
-              return true
-            }).catch((err) => {
-              wiki.logger.error('[AUTH] An error occured while creating root admin account:')
-              wiki.logger.error(err)
-              return err
-            })
-          } else { return true }
-        })
-      } else { return true }
-    })
+  return wiki.db.User.findOne({ provider: 'local', email: 'guest@example.com' }).then((c) => {
+    if (c < 1) {
+      // Create guest account
+
+      return wiki.db.User.create({
+        provider: 'local',
+        email: 'guest@example.com',
+        name: 'Guest',
+        password: '',
+        role: 'guest'
+      }).then(() => {
+        wiki.logger.info('[AUTH] Guest account created successfully!')
+        return true
+      }).catch((err) => {
+        wiki.logger.error('[AUTH] An error occured while creating guest account:')
+        wiki.logger.error(err)
+        return err
+      })
+    }
   })
+
+  // .then(() => {
+  //   if (process.env.WIKI_JS_HEROKU) {
+  //     return wiki.db.User.findOne({ provider: 'local', email: process.env.WIKI_ADMIN_EMAIL }).then((c) => {
+  //       if (c < 1) {
+  //         // Create root admin account (HEROKU ONLY)
+
+  //         return wiki.db.User.create({
+  //           provider: 'local',
+  //           email: process.env.WIKI_ADMIN_EMAIL,
+  //           name: 'Administrator',
+  //           password: '$2a$04$MAHRw785Xe/Jd5kcKzr3D.VRZDeomFZu2lius4gGpZZ9cJw7B7Mna', // admin123 (default)
+  //           role: 'admin'
+  //         }).then(() => {
+  //           wiki.logger.info('[AUTH] Root admin account created successfully!')
+  //           return true
+  //         }).catch((err) => {
+  //           wiki.logger.error('[AUTH] An error occured while creating root admin account:')
+  //           wiki.logger.error(err)
+  //           return err
+  //         })
+  //       } else { return true }
+  //     })
+  //   } else { return true }
+  // })
 }

server/modules/config.js

@@ -8,62 +8,95 @@ const _ = require('lodash')
 const path = require('path')
 const cfgHelper = require('../helpers/config')
 
-/**
- * Load Application Configuration
- *
- * @param      {Object}  confPaths  Path to the configuration files
- * @return     {Object}  Application Configuration
- */
-module.exports = (confPaths) => {
-  confPaths = _.defaults(confPaths, {
-    config: path.join(wiki.ROOTPATH, 'config.yml'),
-    data: path.join(wiki.SERVERPATH, 'app/data.yml'),
-    dataRegex: path.join(wiki.SERVERPATH, 'app/regex.js')
-  })
-
-  let appconfig = {}
-  let appdata = {}
-
-  try {
-    appconfig = yaml.safeLoad(
-      cfgHelper.parseConfigValue(
-        fs.readFileSync(confPaths.config, 'utf8')
+module.exports = {
+  SUBSETS: ['auth', 'features', 'git', 'logging', 'site', 'theme', 'uploads'],
+
+  /**
+   * Load root config from disk
+   *
+   * @param {any} confPaths
+   * @returns
+   */
+  init() {
+    let confPaths = {
+      config: path.join(wiki.ROOTPATH, 'config.yml'),
+      data: path.join(wiki.SERVERPATH, 'app/data.yml'),
+      dataRegex: path.join(wiki.SERVERPATH, 'app/regex.js')
+    }
+
+    let appconfig = {}
+    let appdata = {}
+
+    try {
+      appconfig = yaml.safeLoad(
+        cfgHelper.parseConfigValue(
+          fs.readFileSync(confPaths.config, 'utf8')
+        )
       )
-    )
-    appdata = yaml.safeLoad(fs.readFileSync(confPaths.data, 'utf8'))
-    appdata.regex = require(confPaths.dataRegex)
-  } catch (ex) {
-    console.error(ex)
-    process.exit(1)
-  }
+      appdata = yaml.safeLoad(fs.readFileSync(confPaths.data, 'utf8'))
+      appdata.regex = require(confPaths.dataRegex)
+    } catch (ex) {
+      console.error(ex)
+      process.exit(1)
+    }
 
-  // Merge with defaults
+    // Merge with defaults
 
-  appconfig = _.defaultsDeep(appconfig, appdata.defaults.config)
+    appconfig = _.defaultsDeep(appconfig, appdata.defaults.config)
 
-  // Check port
+    // Check port
 
-  if (appconfig.port < 1) {
-    appconfig.port = process.env.PORT || 80
-  }
+    if (appconfig.port < 1) {
+      appconfig.port = process.env.PORT || 80
+    }
 
   // Convert booleans
 
   appconfig.public = (appconfig.public === true || _.toLower(appconfig.public) === 'true')
 
   // List authentication strategies
+    wiki.config = appconfig
+    wiki.data = appdata
 
-  appconfig.authStrategies = {
-    list: _.filter(appconfig.auth, ['enabled', true]),
-    socialEnabled: (_.chain(appconfig.auth).omit(['local', 'ldap']).filter(['enabled', true]).value().length > 0)
-  }
-  if (appconfig.authStrategies.list.length < 1) {
-    console.error(new Error('You must enable at least 1 authentication strategy!'))
-    process.exit(1)
-  }
+    // List authentication strategies
+
+    // appconfig.authStrategies = {
+    //   list: _.filter(appconfig.auth, ['enabled', true]),
+    //   socialEnabled: (_.chain(appconfig.auth).omit('local').filter(['enabled', true]).value().length > 0)
+    // }
+    // if (appconfig.authStrategies.list.length < 1) {
+    //   console.error(new Error('You must enable at least 1 authentication strategy!'))
+    //   process.exit(1)
+    // }
+  },
+
+  /**
+   * Load config from DB
+   *
+   * @param {Array} subsets Array of subsets to load
+   * @returns Promise
+   */
+  loadFromDb(subsets) {
+    if (!_.isArray(subsets) || subsets.length === 0) {
+      subsets = this.SUBSETS
+    }
 
-  return {
-    config: appconfig,
-    data: appdata
+    return wiki.db.Setting.findAll({
+      attributes: ['key', 'config'],
+      where: {
+        key: {
+          $in: subsets
+        }
+      }
+    }).then(results => {
+      if (_.isArray(results) && results.length > 0) {
+        results.forEach(result => {
+          wiki.config[result.key] = result.config
+        })
+        return true
+      } else {
+        return Promise.reject(new Error('Invalid DB Configuration result set'))
+      }
+    })
   }
 }

server/modules/db.js

@@ -5,6 +5,7 @@
 const fs = require('fs')
 const path = require('path')
 const _ = require('lodash')
+const Promise = require('bluebird')
 
 /**
  * PostgreSQL DB module
@@ -33,7 +34,8 @@ module.exports = {
         max: 10,
         min: 0,
         idle: 10000
-      }
+      },
+      logging: false
     })
 
     // Attempt to connect and authenticate to DB
@@ -63,10 +65,10 @@ module.exports = {
 
     // Sync DB
 
-    self.onReady = self.inst.sync({
+    self.onReady = (wiki.IS_MASTER) ? self.inst.sync({
       force: false,
       logging: false
-    })
+    }) : Promise.resolve()
 
     return self
   }

server/modules/disk.js

@@ -27,7 +27,7 @@ module.exports = {
     this._uploadsThumbsPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data, 'thumbs')
 
     this.createBaseDirectories()
-    this.initMulter()
+    // this.initMulter()
 
     return this
   },
@@ -37,8 +37,10 @@ module.exports = {
    */
   initMulter () {
     let maxFileSizes = {
-      img: wiki.config.uploads.maxImageFileSize * 1024 * 1024,
-      file: wiki.config.uploads.maxOtherFileSize * 1024 * 1024
+      // img: wiki.config.uploads.maxImageFileSize * 1024 * 1024,
+      // file: wiki.config.uploads.maxOtherFileSize * 1024 * 1024
+      img: 3 * 1024 * 1024,
+      file: 10 * 1024 * 1024
     }
 
     // -> IMAGES

server/modules/git.js

@@ -51,7 +51,7 @@ module.exports = {
 
     // -> Initialize repository
 
-    self.onReady = self._initRepo()
+    self.onReady = (wiki.IS_MASTER) ? self._initRepo() : Promise.resolve()
 
     // Define signature
 

server/modules/logger.js

@@ -4,75 +4,77 @@
 
 const cluster = require('cluster')
 
-module.exports = () => {
-  let winston = require('winston')
+module.exports = {
+  init() {
+    let winston = require('winston')
 
-  // Console
+    // Console
 
-  let logger = new (winston.Logger)({
-    level: (wiki.IS_DEBUG) ? 'debug' : 'info',
-    transports: [
-      new (winston.transports.Console)({
-        level: (wiki.IS_DEBUG) ? 'debug' : 'info',
-        prettyPrint: true,
-        colorize: true,
-        silent: false,
-        timestamp: true
-      })
-    ]
-  })
+    let logger = new (winston.Logger)({
+      level: (wiki.IS_DEBUG) ? 'debug' : 'info',
+      transports: [
+        new (winston.transports.Console)({
+          level: (wiki.IS_DEBUG) ? 'debug' : 'info',
+          prettyPrint: true,
+          colorize: true,
+          silent: false,
+          timestamp: true
+        })
+      ]
+    })
 
-  logger.filters.push((level, msg) => {
-    let processName = (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}`
-    return '[' + processName + '] ' + msg
-  })
+    logger.filters.push((level, msg) => {
+      let processName = (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}`
+      return '[' + processName + '] ' + msg
+    })
 
-  // External services
+    // External services
 
-  if (wiki.config.externalLogging.bugsnag) {
-    const bugsnagTransport = require('./winston-transports/bugsnag')
-    logger.add(bugsnagTransport, {
-      level: 'warn',
-      key: wiki.config.externalLogging.bugsnag
-    })
-  }
+    // if (wiki.config.externalLogging.bugsnag) {
+    //   const bugsnagTransport = require('./winston-transports/bugsnag')
+    //   logger.add(bugsnagTransport, {
+    //     level: 'warn',
+    //     key: wiki.config.externalLogging.bugsnag
+    //   })
+    // }
 
-  if (wiki.config.externalLogging.loggly) {
-    require('winston-loggly-bulk')
-    logger.add(winston.transports.Loggly, {
-      token: wiki.config.externalLogging.loggly.token,
-      subdomain: wiki.config.externalLogging.loggly.subdomain,
-      tags: ['wiki-js'],
-      level: 'warn',
-      json: true
-    })
-  }
+    // if (wiki.config.externalLogging.loggly) {
+    //   require('winston-loggly-bulk')
+    //   logger.add(winston.transports.Loggly, {
+    //     token: wiki.config.externalLogging.loggly.token,
+    //     subdomain: wiki.config.externalLogging.loggly.subdomain,
+    //     tags: ['wiki-js'],
+    //     level: 'warn',
+    //     json: true
+    //   })
+    // }
 
-  if (wiki.config.externalLogging.papertrail) {
-    require('winston-papertrail').Papertrail // eslint-disable-line no-unused-expressions
-    logger.add(winston.transports.Papertrail, {
-      host: wiki.config.externalLogging.papertrail.host,
-      port: wiki.config.externalLogging.papertrail.port,
-      level: 'warn',
-      program: 'wiki.js'
-    })
-  }
+    // if (wiki.config.externalLogging.papertrail) {
+    //   require('winston-papertrail').Papertrail // eslint-disable-line no-unused-expressions
+    //   logger.add(winston.transports.Papertrail, {
+    //     host: wiki.config.externalLogging.papertrail.host,
+    //     port: wiki.config.externalLogging.papertrail.port,
+    //     level: 'warn',
+    //     program: 'wiki.js'
+    //   })
+    // }
 
-  if (wiki.config.externalLogging.rollbar) {
-    const rollbarTransport = require('./winston-transports/rollbar')
-    logger.add(rollbarTransport, {
-      level: 'warn',
-      key: wiki.config.externalLogging.rollbar
-    })
-  }
+    // if (wiki.config.externalLogging.rollbar) {
+    //   const rollbarTransport = require('./winston-transports/rollbar')
+    //   logger.add(rollbarTransport, {
+    //     level: 'warn',
+    //     key: wiki.config.externalLogging.rollbar
+    //   })
+    // }
 
-  if (wiki.config.externalLogging.sentry) {
-    const sentryTransport = require('./winston-transports/sentry')
-    logger.add(sentryTransport, {
-      level: 'warn',
-      key: wiki.config.externalLogging.sentry
-    })
-  }
+    // if (wiki.config.externalLogging.sentry) {
+    //   const sentryTransport = require('./winston-transports/sentry')
+    //   logger.add(sentryTransport, {
+    //     level: 'warn',
+    //     key: wiki.config.externalLogging.sentry
+    //   })
+    // }
 
-  return logger
+    return logger
+  }
 }

server/modules/markdown.js

@@ -23,7 +23,8 @@ const mdRemove = require('remove-markdown')
 
 var mkdown = md({
   html: true,
-  breaks: wiki.config.features.linebreaks,
+  // breaks: wiki.config.features.linebreaks,
+  breaks: true,
   linkify: true,
   typography: true,
   highlight(str, lang) {
@@ -57,7 +58,8 @@ var mkdown = md({
   })
   .use(mdAttrs)
 
-if (wiki.config.features.mathjax) {
+// if (wiki.config.features.mathjax) {
+if (true) {
   mkdown.use(mdMathjax)
 }
 

server/modules/uploads-agent.js

 'use strict'
 
-/* global db, git, lang, upl */
+/* global wiki */
 
 const path = require('path')
 const Promise = require('bluebird')
@@ -32,8 +32,8 @@ module.exports = {
   init () {
     let self = this
 
-    self._uploadsPath = path.resolve(ROOTPATH, appconfig.paths.repo, 'uploads')
-    self._uploadsThumbsPath = path.resolve(ROOTPATH, appconfig.paths.data, 'thumbs')
+    self._uploadsPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo, 'uploads')
+    self._uploadsThumbsPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data, 'thumbs')
 
     return self
   },
@@ -59,16 +59,16 @@ module.exports = {
     self._watcher.on('add', (p) => {
       let pInfo = self.parseUploadsRelPath(p)
       return self.processFile(pInfo.folder, pInfo.filename).then((mData) => {
-        return db.UplFile.findByIdAndUpdate(mData._id, mData, { upsert: true })
+        return wiki.db.UplFile.findByIdAndUpdate(mData._id, mData, { upsert: true })
       }).then(() => {
-        return git.commitUploads(lang.t('git:uploaded', { path: p }))
+        return wiki.git.commitUploads(wiki.lang.t('git:uploaded', { path: p }))
       })
     })
 
     // -> Remove upload file
 
     self._watcher.on('unlink', (p) => {
-      return git.commitUploads(lang.t('git:deleted', { path: p }))
+      return wiki.git.commitUploads(wiki.lang.t('git:deleted', { path: p }))
     })
   },
 
@@ -91,8 +91,8 @@ module.exports = {
 
         // Add folders to DB
 
-        return db.UplFolder.remove({}).then(() => {
-          return db.UplFolder.insertMany(_.map(folderNames, (f) => {
+        return wiki.db.UplFolder.remove({}).then(() => {
+          return wiki.db.UplFolder.insertMany(_.map(folderNames, (f) => {
             return {
               _id: 'f:' + f,
               name: f
@@ -107,7 +107,7 @@ module.exports = {
             let fldPath = path.join(self._uploadsPath, fldName)
             return fs.readdirAsync(fldPath).then((fList) => {
               return Promise.map(fList, (f) => {
-                return upl.processFile(fldName, f).then((mData) => {
+                return wiki.upl.processFile(fldName, f).then((mData) => {
                   if (mData) {
                     allFiles.push(mData)
                   }
@@ -118,9 +118,9 @@ module.exports = {
           }, {concurrency: 1}).finally(() => {
             // Add files to DB
 
-            return db.UplFile.remove({}).then(() => {
+            return wiki.db.UplFile.remove({}).then(() => {
               if (_.isArray(allFiles) && allFiles.length > 0) {
-                return db.UplFile.insertMany(allFiles)
+                return wiki.db.UplFile.insertMany(allFiles)
               } else {
                 return true
               }
@@ -131,7 +131,7 @@ module.exports = {
     }).then(() => {
       // Watch for new changes
 
-      return upl.watch()
+      return wiki.upl.watch()
     })
   },
 

server/queues/git-sync.js

+'use strict'
+
+/* global wiki */
+
+const Promise = require('bluebird')
+const fs = Promise.promisifyAll(require('fs-extra'))
+const klaw = require('klaw')
+const moment = require('moment')
+const path = require('path')
+const entryHelper = require('../helpers/entry')
+
+module.exports = (job, done) => {
+  return wiki.git.resync().then(() => {
+    // -> Stream all documents
+
+    let cacheJobs = []
+    let jobCbStreamDocsResolve = null
+    let jobCbStreamDocs = new Promise((resolve, reject) => {
+      jobCbStreamDocsResolve = resolve
+    })
+
+    klaw(wiki.REPOPATH).on('data', function (item) {
+      if (path.extname(item.path) === '.md' && path.basename(item.path) !== 'README.md') {
+        let entryPath = entryHelper.parsePath(entryHelper.getEntryPathFromFullPath(item.path))
+        let cachePath = entryHelper.getCachePath(entryPath)
+
+        // -> Purge outdated cache
+
+        cacheJobs.push(
+          fs.statAsync(cachePath).then((st) => {
+            return moment(st.mtime).isBefore(item.stats.mtime) ? 'expired' : 'active'
+          }).catch((err) => {
+            return (err.code !== 'EEXIST') ? err : 'new'
+          }).then((fileStatus) => {
+            // -> Delete expired cache file
+
+            if (fileStatus === 'expired') {
+              return fs.unlinkAsync(cachePath).return(fileStatus)
+            }
+
+            return fileStatus
+          }).then((fileStatus) => {
+            // -> Update cache and search index
+
+            if (fileStatus !== 'active') {
+              return global.entries.updateCache(entryPath).then(entry => {
+                process.send({
+                  action: 'searchAdd',
+                  content: entry
+                })
+                return true
+              })
+            }
+
+            return true
+          })
+        )
+      }
+    }).on('end', () => {
+      jobCbStreamDocsResolve(Promise.all(cacheJobs))
+    })
+
+    return jobCbStreamDocs
+  })
+}

server/queues/upl-clear-temp.js

+'use strict'
+
+/* global wiki */
+
+const Promise = require('bluebird')
+const fs = Promise.promisifyAll(require('fs-extra'))
+const moment = require('moment')
+const path = require('path')
+
+module.exports = (job, done) => {
+  return fs.readdirAsync(wiki.UPLTEMPPATH).then((ls) => {
+    let fifteenAgo = moment().subtract(15, 'minutes')
+
+    return Promise.map(ls, (f) => {
+      return fs.statAsync(path.join(wiki.UPLTEMPPATH, f)).then((s) => { return { filename: f, stat: s } })
+    }).filter((s) => { return s.stat.isFile() }).then((arrFiles) => {
+      return Promise.map(arrFiles, (f) => {
+        if (moment(f.stat.ctime).isBefore(fifteenAgo, 'minute')) {
+          return fs.unlinkAsync(path.join(wiki.UPLTEMPPATH, f.filename))
+        } else {
+          return true
+        }
+      })
+    })
+  })
+}

server/worker.js

@@ -4,30 +4,27 @@
 
 const path = require('path')
 
+wiki.REPOPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo)
+wiki.DATAPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.data)
+wiki.UPLTEMPPATH = path.join(wiki.DATAPATH, 'temp-upload')
+
 // ----------------------------------------
 // Load global modules
 // ----------------------------------------
 
-wiki.db = require('./modules/db').init()
-wiki.upl = require('./modules/uploads-agent').init()
-wiki.git = require('./modules/git').init()
-wiki.entries = require('./modules/entries').init()
+// wiki.upl = require('./modules/uploads-agent').init()
+// wiki.git = require('./modules/git').init()
+// wiki.entries = require('./modules/entries').init()
 wiki.lang = require('i18next')
 wiki.mark = require('./modules/markdown')
 
 // ----------------------------------------
-// Load modules
+// Load local modules
 // ----------------------------------------
 
-const moment = require('moment')
 const Promise = require('bluebird')
-const fs = Promise.promisifyAll(require('fs-extra'))
-const klaw = require('klaw')
-const Cron = require('cron').CronJob
 const i18nBackend = require('i18next-node-fs-backend')
 
-const entryHelper = require('./helpers/entry')
-
 // ----------------------------------------
 // Localization Engine
 // ----------------------------------------
@@ -46,143 +43,21 @@ wiki.lang.use(i18nBackend).init({
 })
 
 // ----------------------------------------
-// Start Cron
+// Start Queues
 // ----------------------------------------
 
-let job
-let jobIsBusy = false
-let jobUplWatchStarted = false
-
-wiki.db.onReady.then(() => {
-  return wiki.db.Entry.remove({})
-}).then(() => {
-  job = new Cron({
-    cronTime: '0 */5 * * * *',
-    onTick: () => {
-      // Make sure we don't start two concurrent jobs
-
-      if (jobIsBusy) {
-        wiki.logger.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)')
-        return
-      }
-      wiki.logger.info('Running all jobs...')
-      jobIsBusy = true
-
-      // Prepare async job collector
-
-      let jobs = []
-      let repoPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo)
-      let dataPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data)
-      let uploadsTempPath = path.join(dataPath, 'temp-upload')
-
-      // ----------------------------------------
-      // REGULAR JOBS
-      // ----------------------------------------
-
-      //* ****************************************
-      // -> Sync with Git remote
-      //* ****************************************
-
-      jobs.push(wiki.git.resync().then(() => {
-        // -> Stream all documents
-
-        let cacheJobs = []
-        let jobCbStreamDocsResolve = null
-        let jobCbStreamDocs = new Promise((resolve, reject) => {
-          jobCbStreamDocsResolve = resolve
-        })
-
-        klaw(repoPath).on('data', function (item) {
-          if (path.extname(item.path) === '.md' && path.basename(item.path) !== 'README.md') {
-            let entryPath = entryHelper.parsePath(entryHelper.getEntryPathFromFullPath(item.path))
-            let cachePath = entryHelper.getCachePath(entryPath)
-
-            // -> Purge outdated cache
-
-            cacheJobs.push(
-              fs.statAsync(cachePath).then((st) => {
-                return moment(st.mtime).isBefore(item.stats.mtime) ? 'expired' : 'active'
-              }).catch((err) => {
-                return (err.code !== 'EEXIST') ? err : 'new'
-              }).then((fileStatus) => {
-                // -> Delete expired cache file
+const Bull = require('bull')
+const autoload = require('auto-load')
 
-                if (fileStatus === 'expired') {
-                  return fs.unlinkAsync(cachePath).return(fileStatus)
-                }
+let queues = autoload(path.join(wiki.SERVERPATH, 'queues'))
 
-                return fileStatus
-              }).then((fileStatus) => {
-                // -> Update cache and search index
-
-                if (fileStatus !== 'active') {
-                  return global.entries.updateCache(entryPath).then(entry => {
-                    process.send({
-                      action: 'searchAdd',
-                      content: entry
-                    })
-                    return true
-                  })
-                }
-
-                return true
-              })
-            )
-          }
-        }).on('end', () => {
-          jobCbStreamDocsResolve(Promise.all(cacheJobs))
-        })
-
-        return jobCbStreamDocs
-      }))
-
-      //* ****************************************
-      // -> Clear failed temporary upload files
-      //* ****************************************
-
-      jobs.push(
-        fs.readdirAsync(uploadsTempPath).then((ls) => {
-          let fifteenAgo = moment().subtract(15, 'minutes')
-
-          return Promise.map(ls, (f) => {
-            return fs.statAsync(path.join(uploadsTempPath, f)).then((s) => { return { filename: f, stat: s } })
-          }).filter((s) => { return s.stat.isFile() }).then((arrFiles) => {
-            return Promise.map(arrFiles, (f) => {
-              if (moment(f.stat.ctime).isBefore(fifteenAgo, 'minute')) {
-                return fs.unlinkAsync(path.join(uploadsTempPath, f.filename))
-              } else {
-                return true
-              }
-            })
-          })
-        })
-      )
-
-      // ----------------------------------------
-      // Run
-      // ----------------------------------------
-
-      Promise.all(jobs).then(() => {
-        wiki.logger.info('All jobs completed successfully! Going to sleep for now.')
-
-        if (!jobUplWatchStarted) {
-          jobUplWatchStarted = true
-          wiki.upl.initialScan().then(() => {
-            job.start()
-          })
-        }
-
-        return true
-      }).catch((err) => {
-        wiki.logger.error('One or more jobs have failed: ', err)
-      }).finally(() => {
-        jobIsBusy = false
-      })
-    },
-    start: false,
-    timeZone: 'UTC',
-    runOnInit: true
-  })
+Promise.join(
+  wiki.db.onReady
+  // wiki.upl.initialScan()
+).then(() => {
+  for (let queueName in queues) {
+    new Bull(queueName, { redis: wiki.config.redis }).process(queues[queueName])
+  }
 })
 
 // ----------------------------------------
@@ -190,11 +65,6 @@ wiki.db.onReady.then(() => {
 // ----------------------------------------
 
 process.on('disconnect', () => {
-  wiki.logger.warn('Lost connection to main server. Exiting...')
-  job.stop()
+  wiki.logger.warn('Lost connection to Master. Exiting...')
   process.exit()
 })
-
-process.on('exit', () => {
-  job.stop()
-})