Skip to content

W
wiki-js
Commit 9c112ab5 authored by NGPixel's avatar NGPixel
Browse files
Options

feat: cluster implementation

parent 60750eee
Hide whitespace changes
Inline Side-by-side
Showing with 354 additions and 561 deletions
config.sample.yml
View file @ 9c112ab5
...@@ -5,22 +5,8 @@ ...@@ -5,22 +5,8 @@
# https://docs.requarks.io/wiki/install # https://docs.requarks.io/wiki/install
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# Title of this site
# ---------------------------------------------------------------------
title: Wiki
# ---------------------------------------------------------------------
# Full public path to the site, without the trailing slash
# ---------------------------------------------------------------------
# INCLUDE CLIENT PORT IF NOT 80/443!
host: http://localhost
# ---------------------------------------------------------------------
# Port the main server should listen to (80 by default) # Port the main server should listen to (80 by default)
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# To use process.env.PORT, comment the line below:
port: 80 port: 80
...@@ -33,136 +19,23 @@ paths: ...@@ -33,136 +19,23 @@ paths:
data: ./data data: ./data
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# Upload Limits # Database
# ---------------------------------------------------------------------
# In megabytes (MB)
uploads:
maxImageFileSize: 3
maxOtherFileSize: 100
# ---------------------------------------------------------------------
# Site Language
# ---------------------------------------------------------------------
# Possible values: en, es, fr, ko, ru or zh
lang: en
# ---------------------------------------------------------------------
# Site Authentication
# ---------------------------------------------------------------------
public: false
auth:
defaultReadAccess: false
local:
enabled: true
google:
enabled: true
clientId: GOOGLE_CLIENT_ID
clientSecret: GOOGLE_CLIENT_SECRET
microsoft:
enabled: true
clientId: MS_APP_ID
clientSecret: MS_APP_SECRET
facebook:
enabled: false
clientId: FACEBOOK_APP_ID
clientSecret: FACEBOOK_APP_SECRET
github:
enabled: false
clientId: GITHUB_CLIENT_ID
clientSecret: GITHUB_CLIENT_SECRET
slack:
enabled: false
clientId: SLACK_CLIENT_ID
clientSecret: SLACK_CLIENT_SECRET
ldap:
enabled: false
url: ldap://serverhost:389
bindDn: cn='root'
bindCredentials: BIND_PASSWORD
searchBase: o=users,o=example.com
searchFilter: (uid={{username}})
tlsEnabled: false
tlsCertPath: C:\example\root_ca_cert.crt
azure:
enabled: false
clientID: APP_ID
clientSecret: APP_SECRET_KEY
resource: '00000002-0000-0000-c000-000000000000'
tenant: 'YOUR_TENANT.onmicrosoft.com'
# ---------------------------------------------------------------------
# Secret key to use when encrypting sessions
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# Use a long and unique random string (256-bit keys are perfect!)
sessionSecret: 1234567890abcdefghijklmnopqrstuvxyz db:
host: localhost
port: 5432
user: wikijs
pass: wikijsrocks
db: wiki
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# Database Connection String # Redis
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# You can also use an ENV variable by using $ENV_VAR_NAME as the value
db: mongodb://localhost:27017/wiki redis:
host: localhost
# --------------------------------------------------------------------- port: 6379
# Git Connection Info db: 0
# --------------------------------------------------------------------- password: null
git:
url: https://github.com/Organization/Repo
branch: master
auth:
# Type: basic or ssh
type: ssh
# Only for Basic authentication:
username: marty
password: MartyMcFly88
# Only for SSH authentication:
privateKey: /etc/wiki/keys/git.pem
sslVerify: true
# Default email to use as commit author
serverEmail: marty@example.com
# Whether to use user email as author in commits
showUserEmail: true
# ---------------------------------------------------------------------
# Features
# ---------------------------------------------------------------------
# You can enable / disable specific features below
features:
linebreaks: true
mathjax: true
# ---------------------------------------------------------------------
# External Logging
# ---------------------------------------------------------------------
externalLogging:
bugsnag: false
loggly: false
papertrail: false
rollbar: false
sentry: false
# ---------------------------------------------------------------------
# Color Theme
# ---------------------------------------------------------------------
theme:
primary: indigo
alt: blue-grey
footer: blue-grey
code:
dark: true
colorize: true
package.json
View file @ 9c112ab5
...@@ -43,6 +43,7 @@ ...@@ -43,6 +43,7 @@
"bcryptjs-then": "~1.0.1", "bcryptjs-then": "~1.0.1",
"bluebird": "~3.5.0", "bluebird": "~3.5.0",
"body-parser": "~1.17.2", "body-parser": "~1.17.2",
"bull": "/home/nick3.0.0-rc.4",
"bunyan": "~1.8.10", "bunyan": "~1.8.10",
"cheerio": "~1.0.0-rc.2", "cheerio": "~1.0.0-rc.2",
"child-process-promise": "~2.2.1", "child-process-promise": "~2.2.1",
......
server/index.js
View file @ 9c112ab5
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
// =========================================== // ===========================================
// Wiki.js // Wiki.js
// 1.0.1
// Licensed under AGPLv3 // Licensed under AGPLv3
// =========================================== // ===========================================
...@@ -28,251 +27,28 @@ wiki.data = appconf.data ...@@ -28,251 +27,28 @@ wiki.data = appconf.data
// Load Winston // Load Winston
// ---------------------------------------- // ----------------------------------------
wiki.logger = require('./modules/logger')(wiki.IS_DEBUG, 'SERVER') wiki.logger = require('./modules/logger')()
wiki.logger.info('Wiki.js is initializing...')
// ---------------------------------------- // ----------------------------------------
// Load global modules // Start Cluster
// ---------------------------------------- // ----------------------------------------
wiki.disk = require('./modules/disk').init() const cluster = require('cluster')
wiki.db = require('./modules/db').init() const numCPUs = require('os').cpus().length
wiki.entries = require('./modules/entries').init()
wiki.git = require('./modules/git').init(false)
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
wiki.redis = require('./modules/redis').init()
wiki.search = require('./modules/search').init()
wiki.upl = require('./modules/uploads').init()
// ---------------------------------------- if (cluster.isMaster) {
// Load modules wiki.logger.info('Wiki.js is initializing...')
// ----------------------------------------
const autoload = require('auto-load')
const bodyParser = require('body-parser')
const compression = require('compression')
const cookieParser = require('cookie-parser')
const express = require('express')
const favicon = require('serve-favicon')
const flash = require('connect-flash')
const fork = require('child_process').fork
const http = require('http')
const i18nBackend = require('i18next-node-fs-backend')
const passport = require('passport')
const passportSocketIo = require('passport.socketio')
const session = require('express-session')
const SessionRedisStore = require('connect-redis')(session)
const graceful = require('node-graceful')
const socketio = require('socket.io')
const graphqlApollo = require('apollo-server-express')
const graphqlSchema = require('./modules/graphql')
var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares'))
var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers'))
// ----------------------------------------
// Define Express App
// ----------------------------------------
const app = express()
wiki.app = app
app.use(compression())
// ----------------------------------------
// Security
// ----------------------------------------
app.use(mw.security)
// ----------------------------------------
// Public Assets
// ----------------------------------------
app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
index: false,
maxAge: '7d'
}))
// ----------------------------------------
// Passport Authentication
// ----------------------------------------
require('./modules/auth')(passport)
wiki.rights = require('./modules/rights')
wiki.rights.init()
let sessionStore = new SessionRedisStore({
client: wiki.redis
})
app.use(cookieParser())
app.use(session({
name: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
resave: false,
saveUninitialized: false
}))
app.use(flash())
app.use(passport.initialize())
app.use(passport.session())
// ----------------------------------------
// SEO
// ----------------------------------------
app.use(mw.seo)
// ----------------------------------------
// Localization Engine
// ----------------------------------------
wiki.lang.use(i18nBackend).init({
load: 'languageOnly',
ns: ['common', 'admin', 'auth', 'errors', 'git'],
defaultNS: 'common',
saveMissing: false,
preload: [wiki.config.lang],
lng: wiki.config.lang,
fallbackLng: 'en',
backend: {
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
}
})
// ----------------------------------------
// View Engine Setup
// ----------------------------------------
app.set('views', path.join(wiki.SERVERPATH, 'views'))
app.set('view engine', 'pug')
app.use(bodyParser.json({ limit: '1mb' }))
app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
// ----------------------------------------
// View accessible data
// ----------------------------------------
app.locals._ = require('lodash')
app.locals.t = wiki.lang.t.bind(wiki.lang)
app.locals.moment = require('moment')
app.locals.moment.locale(wiki.config.lang)
app.locals.appconfig = wiki.config
app.use(mw.flash)
// ----------------------------------------
// Controllers
// ----------------------------------------
app.use('/', ctrl.auth)
app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
app.use('/uploads', mw.auth, ctrl.uploads)
app.use('/admin', mw.auth, ctrl.admin)
app.use('/', mw.auth, ctrl.pages)
// ----------------------------------------
// Error handling
// ----------------------------------------
app.use(function (req, res, next) {
var err = new Error('Not Found')
err.status = 404
next(err)
})
app.use(function (err, req, res, next) {
res.status(err.status || 500)
res.render('error', {
message: err.message,
error: wiki.IS_DEBUG ? err : {}
})
})
// ----------------------------------------
// Start HTTP server
// ----------------------------------------
wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
app.set('port', wiki.config.port)
var server = http.createServer(app)
var io = socketio(server)
server.listen(wiki.config.port)
server.on('error', (error) => {
if (error.syscall !== 'listen') {
throw error
}
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
return process.exit(1)
case 'EADDRINUSE':
wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
return process.exit(1)
default:
throw error
}
})
server.on('listening', () => { require('./master')
wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
})
// ----------------------------------------
// WebSocket
// ----------------------------------------
io.use(passportSocketIo.authorize({ for (let i = 0; i < numCPUs; i++) {
key: 'wikijs.sid', cluster.fork()
store: sessionStore,
secret: wiki.config.sessionSecret,
cookieParser,
success: (data, accept) => {
accept()
},
fail: (data, message, error, accept) => {
accept()
} }
}))
io.on('connection', ctrl.ws)
// ----------------------------------------
// Start child processes
// ----------------------------------------
let bgAgent = fork(path.join(wiki.SERVERPATH, 'agent.js')) cluster.on('exit', (worker, code, signal) => {
wiki.logger.info(`Worker #${worker.id} died.`)
bgAgent.on('message', m => {
if (!m.action) {
return
}
switch (m.action) {
case 'searchAdd':
wiki.search.add(m.content)
break
}
})
// ----------------------------------------
// Graceful shutdown
// ----------------------------------------
graceful.on('exit', () => {
wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
bgAgent.kill()
wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
return global.git.resync().then(() => {
wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
process.exit()
}) })
}) } else {
wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`)
// require('./worker')
}
server/master.js 0 → 100644
View file @ 9c112ab5
'use strict'
/* global wiki */
const path = require('path')
// ----------------------------------------
// Load global modules
// ----------------------------------------
wiki.disk = require('./modules/disk').init()
wiki.db = require('./modules/db').init()
wiki.entries = require('./modules/entries').init()
wiki.git = require('./modules/git').init(false)
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
wiki.redis = require('./modules/redis').init()
wiki.search = require('./modules/search').init()
wiki.upl = require('./modules/uploads').init()
// ----------------------------------------
// Load modules
// ----------------------------------------
const autoload = require('auto-load')
const bodyParser = require('body-parser')
const compression = require('compression')
const cookieParser = require('cookie-parser')
const express = require('express')
const favicon = require('serve-favicon')
const flash = require('connect-flash')
const http = require('http')
const i18nBackend = require('i18next-node-fs-backend')
const passport = require('passport')
const passportSocketIo = require('passport.socketio')
const session = require('express-session')
const SessionRedisStore = require('connect-redis')(session)
const graceful = require('node-graceful')
const socketio = require('socket.io')
const graphqlApollo = require('apollo-server-express')
const graphqlSchema = require('./modules/graphql')
var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares'))
var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers'))
// ----------------------------------------
// Define Express App
// ----------------------------------------
const app = express()
wiki.app = app
app.use(compression())
// ----------------------------------------
// Security
// ----------------------------------------
app.use(mw.security)
// ----------------------------------------
// Public Assets
// ----------------------------------------
app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
index: false,
maxAge: '7d'
}))
// ----------------------------------------
// Passport Authentication
// ----------------------------------------
require('./modules/auth')(passport)
wiki.rights = require('./modules/rights')
wiki.rights.init()
let sessionStore = new SessionRedisStore({
client: wiki.redis
})
app.use(cookieParser())
app.use(session({
name: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
resave: false,
saveUninitialized: false
}))
app.use(flash())
app.use(passport.initialize())
app.use(passport.session())
// ----------------------------------------
// SEO
// ----------------------------------------
app.use(mw.seo)
// ----------------------------------------
// Localization Engine
// ----------------------------------------
wiki.lang.use(i18nBackend).init({
load: 'languageOnly',
ns: ['common', 'admin', 'auth', 'errors', 'git'],
defaultNS: 'common',
saveMissing: false,
preload: [wiki.config.lang],
lng: wiki.config.lang,
fallbackLng: 'en',
backend: {
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
}
})
// ----------------------------------------
// View Engine Setup
// ----------------------------------------
app.set('views', path.join(wiki.SERVERPATH, 'views'))
app.set('view engine', 'pug')
app.use(bodyParser.json({ limit: '1mb' }))
app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
// ----------------------------------------
// View accessible data
// ----------------------------------------
app.locals._ = require('lodash')
app.locals.t = wiki.lang.t.bind(wiki.lang)
app.locals.moment = require('moment')
app.locals.moment.locale(wiki.config.lang)
app.locals.appconfig = wiki.config
app.use(mw.flash)
// ----------------------------------------
// Controllers
// ----------------------------------------
app.use('/', ctrl.auth)
app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
app.use('/uploads', mw.auth, ctrl.uploads)
app.use('/admin', mw.auth, ctrl.admin)
app.use('/', mw.auth, ctrl.pages)
// ----------------------------------------
// Error handling
// ----------------------------------------
app.use(function (req, res, next) {
var err = new Error('Not Found')
err.status = 404
next(err)
})
app.use(function (err, req, res, next) {
res.status(err.status || 500)
res.render('error', {
message: err.message,
error: wiki.IS_DEBUG ? err : {}
})
})
// ----------------------------------------
// Start HTTP server
// ----------------------------------------
wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
app.set('port', wiki.config.port)
var server = http.createServer(app)
var io = socketio(server)
server.listen(wiki.config.port)
server.on('error', (error) => {
if (error.syscall !== 'listen') {
throw error
}
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
return process.exit(1)
case 'EADDRINUSE':
wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
return process.exit(1)
default:
throw error
}
})
server.on('listening', () => {
wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
})
// ----------------------------------------
// WebSocket
// ----------------------------------------
io.use(passportSocketIo.authorize({
key: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
cookieParser,
success: (data, accept) => {
accept()
},
fail: (data, message, error, accept) => {
accept()
}
}))
io.on('connection', ctrl.ws)
// ----------------------------------------
// Graceful shutdown
// ----------------------------------------
graceful.on('exit', () => {
// wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
// bgAgent.kill()
wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
return global.git.resync().then(() => {
wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
process.exit()
})
})
server/models/_relations.js
View file @ 9c112ab5
...@@ -4,7 +4,10 @@ ...@@ -4,7 +4,10 @@
* Associate DB Model relations * Associate DB Model relations
*/ */
module.exports = db => { module.exports = db => {
db.User.belongsToMany(db.Group, { through: 'UserGroups' }) db.User.belongsToMany(db.Group, { through: 'userGroups' })
db.Group.hasMany(db.Right, { as: 'GroupRights' }) db.Group.hasMany(db.Right, { as: 'groupRights' })
db.Document.hasMany(db.Tag, { as: 'documentTags' })
db.File.belongsTo(db.Folder) db.File.belongsTo(db.Folder)
db.Comment.belongsTo(db.Document)
db.Comment.belongsTo(db.User, { as: 'author' })
} }
server/models/comment.js 0 → 100644
View file @ 9c112ab5
'use strict'
/**
* Comment schema
*/
module.exports = (sequelize, DataTypes) => {
let commentSchema = sequelize.define('comment', {
content: {
type: DataTypes.STRING,
allowNull: false
}
}, {
timestamps: true,
version: true
})
return commentSchema
}
server/models/document.js
View file @ 9c112ab5
...@@ -40,6 +40,11 @@ module.exports = (sequelize, DataTypes) => { ...@@ -40,6 +40,11 @@ module.exports = (sequelize, DataTypes) => {
type: DataTypes.BOOLEAN, type: DataTypes.BOOLEAN,
allowNull: false, allowNull: false,
defaultValue: false defaultValue: false
},
searchContent: {
type: DataTypes.TEXT,
allowNull: true,
defaultValue: ''
} }
}, { }, {
timestamps: true, timestamps: true,
......
server/models/tag.js 0 → 100644
View file @ 9c112ab5
'use strict'
/**
* Tags schema
*/
module.exports = (sequelize, DataTypes) => {
let tagSchema = sequelize.define('tag', {
key: {
type: DataTypes.STRING,
allowNull: false
}
}, {
timestamps: true,
version: true,
indexes: [
{
unique: true,
fields: ['key']
}
]
})
return tagSchema
}
server/modules/db.js
View file @ 9c112ab5
...@@ -64,7 +64,8 @@ module.exports = { ...@@ -64,7 +64,8 @@ module.exports = {
// Sync DB // Sync DB
self.onReady = self.inst.sync({ self.onReady = self.inst.sync({
force: false force: false,
logging: false
}) })
return self return self
......
server/modules/logger.js
View file @ 9c112ab5
...@@ -2,12 +2,10 @@ ...@@ -2,12 +2,10 @@
/* global wiki */ /* global wiki */
module.exports = (processName) => { const cluster = require('cluster')
let winston = require('winston')
if (typeof processName === 'undefined') { module.exports = () => {
processName = 'SERVER' let winston = require('winston')
}
// Console // Console
...@@ -25,6 +23,7 @@ module.exports = (processName) => { ...@@ -25,6 +23,7 @@ module.exports = (processName) => {
}) })
logger.filters.push((level, msg) => { logger.filters.push((level, msg) => {
let processName = (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}`
return '[' + processName + '] ' + msg return '[' + processName + '] ' + msg
}) })
......
server/modules/search-index/index.js deleted 100644 → 0
View file @ 60750eee
const bunyan = require('bunyan')
const level = require('levelup')
const down = require('memdown')
const SearchIndexAdder = require('search-index-adder')
const SearchIndexSearcher = require('search-index-searcher')
module.exports = function (givenOptions, moduleReady) {
const optionsLoaded = function (err, SearchIndex) {
const siUtil = require('./siUtil.js')(SearchIndex.options)
if (err) return moduleReady(err)
SearchIndex.close = siUtil.close
SearchIndex.countDocs = siUtil.countDocs
getAdder(SearchIndex, adderLoaded)
}
const adderLoaded = function (err, SearchIndex) {
if (err) return moduleReady(err)
getSearcher(SearchIndex, searcherLoaded)
}
const searcherLoaded = function (err, SearchIndex) {
if (err) return moduleReady(err)
return moduleReady(err, SearchIndex)
}
getOptions(givenOptions, optionsLoaded)
}
const getAdder = function (SearchIndex, done) {
SearchIndexAdder(SearchIndex.options, function (err, searchIndexAdder) {
SearchIndex.add = searchIndexAdder.add
SearchIndex.callbackyAdd = searchIndexAdder.concurrentAdd // deprecated
SearchIndex.concurrentAdd = searchIndexAdder.concurrentAdd
SearchIndex.createWriteStream = searchIndexAdder.createWriteStream
SearchIndex.dbWriteStream = searchIndexAdder.dbWriteStream
SearchIndex.defaultPipeline = searchIndexAdder.defaultPipeline
SearchIndex.del = searchIndexAdder.deleter
SearchIndex.deleteStream = searchIndexAdder.deleteStream
SearchIndex.flush = searchIndexAdder.flush
done(err, SearchIndex)
})
}
const getSearcher = function (SearchIndex, done) {
SearchIndexSearcher(SearchIndex.options, function (err, searchIndexSearcher) {
SearchIndex.availableFields = searchIndexSearcher.availableFields
SearchIndex.buckets = searchIndexSearcher.bucketStream
SearchIndex.categorize = searchIndexSearcher.categoryStream
SearchIndex.dbReadStream = searchIndexSearcher.dbReadStream
SearchIndex.get = searchIndexSearcher.get
SearchIndex.match = searchIndexSearcher.match
SearchIndex.scan = searchIndexSearcher.scan
SearchIndex.search = searchIndexSearcher.search
SearchIndex.totalHits = searchIndexSearcher.totalHits
done(err, SearchIndex)
})
}
const getOptions = function (options, done) {
var SearchIndex = {}
SearchIndex.options = Object.assign({}, {
indexPath: 'si',
keySeparator: '￮',
logLevel: 'error'
}, options)
options.log = bunyan.createLogger({
name: 'search-index',
level: options.logLevel
})
if (!options.indexes) {
level(SearchIndex.options.indexPath || 'si', {
valueEncoding: 'json',
db: down
}, function (err, db) {
SearchIndex.options.indexes = db
return done(err, SearchIndex)
})
} else {
return done(null, SearchIndex)
}
}
server/modules/search-index/siUtil.js deleted 100644 → 0
View file @ 60750eee
'use strict'
module.exports = function (siOptions) {
var siUtil = {}
siUtil.countDocs = function (callback) {
var count = 0
const gte = 'DOCUMENT' + siOptions.keySeparator
const lte = 'DOCUMENT' + siOptions.keySeparator + siOptions.keySeparator
siOptions.indexes.createReadStream({gte: gte, lte: lte})
.on('data', function (data) {
count++
})
.on('error', function (err) {
return callback(err, null)
})
.on('end', function () {
return callback(null, count)
})
}
siUtil.close = function (callback) {
siOptions.indexes.close(function (err) {
while (!siOptions.indexes.isClosed()) {
// log not always working here- investigate
if (siOptions.log) siOptions.log.info('closing...')
}
if (siOptions.indexes.isClosed()) {
if (siOptions.log) siOptions.log.info('closed...')
callback(err)
}
})
}
return siUtil
}
server/modules/search.js
View file @ 9c112ab5
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
const Promise = require('bluebird') const Promise = require('bluebird')
const _ = require('lodash') const _ = require('lodash')
const searchIndex = require('./search-index') // const searchIndex = require('./search-index')
const stopWord = require('stopword') const stopWord = require('stopword')
const streamToPromise = require('stream-to-promise') const streamToPromise = require('stream-to-promise')
const searchAllowedChars = new RegExp('[^a-z0-9' + wiki.data.regex.cjk + wiki.data.regex.arabic + ' ]', 'g') const searchAllowedChars = new RegExp('[^a-z0-9' + wiki.data.regex.cjk + wiki.data.regex.arabic + ' ]', 'g')
...@@ -22,7 +22,7 @@ module.exports = { ...@@ -22,7 +22,7 @@ module.exports = {
init () { init () {
let self = this let self = this
self._isReady = new Promise((resolve, reject) => { self._isReady = new Promise((resolve, reject) => {
searchIndex({ /*searchIndex({
deletable: true, deletable: true,
fieldedSearch: true, fieldedSearch: true,
indexPath: 'wiki', indexPath: 'wiki',
...@@ -39,7 +39,7 @@ module.exports = { ...@@ -39,7 +39,7 @@ module.exports = {
resolve(true) resolve(true)
}) })
} }
}) }) */
}) })
return self return self
......
server/agent.js → server/worker.js
View file @ 9c112ab5
// =========================================== 'use strict'
// Wiki.js - Background Agent
// 1.0.1
// Licensed under AGPLv3
// ===========================================
const path = require('path') /* global wiki */
const ROOTPATH = process.cwd()
const SERVERPATH = path.join(ROOTPATH, 'server')
global.ROOTPATH = ROOTPATH
global.SERVERPATH = SERVERPATH
const IS_DEBUG = process.env.NODE_ENV === 'development'
let appconf = require('./modules/config')()
global.appconfig = appconf.config
global.appdata = appconf.data
// ----------------------------------------
// Load Winston
// ----------------------------------------
global.winston = require('./modules/logger')(IS_DEBUG, 'AGENT') const path = require('path')
// ---------------------------------------- // ----------------------------------------
// Load global modules // Load global modules
// ---------------------------------------- // ----------------------------------------
global.winston.info('Background Agent is initializing...') wiki.db = require('./modules/db').init()
wiki.upl = require('./modules/uploads-agent').init()
global.db = require('./modules/db').init() wiki.git = require('./modules/git').init()
global.upl = require('./modules/uploads-agent').init() wiki.entries = require('./modules/entries').init()
global.git = require('./modules/git').init() wiki.lang = require('i18next')
global.entries = require('./modules/entries').init() wiki.mark = require('./modules/markdown')
global.lang = require('i18next')
global.mark = require('./modules/markdown')
// ---------------------------------------- // ----------------------------------------
// Load modules // Load modules
...@@ -52,20 +32,18 @@ const entryHelper = require('./helpers/entry') ...@@ -52,20 +32,18 @@ const entryHelper = require('./helpers/entry')
// Localization Engine // Localization Engine
// ---------------------------------------- // ----------------------------------------
global.lang wiki.lang.use(i18nBackend).init({
.use(i18nBackend) load: 'languageOnly',
.init({ ns: ['common', 'admin', 'auth', 'errors', 'git'],
load: 'languageOnly', defaultNS: 'common',
ns: ['common', 'admin', 'auth', 'errors', 'git'], saveMissing: false,
defaultNS: 'common', preload: [wiki.config.lang],
saveMissing: false, lng: wiki.config.lang,
preload: [appconfig.lang], fallbackLng: 'en',
lng: appconfig.lang, backend: {
fallbackLng: 'en', loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
backend: { }
loadPath: path.join(SERVERPATH, 'locales/{{lng}}/{{ns}}.json') })
}
})
// ---------------------------------------- // ----------------------------------------
// Start Cron // Start Cron
...@@ -75,8 +53,8 @@ let job ...@@ -75,8 +53,8 @@ let job
let jobIsBusy = false let jobIsBusy = false
let jobUplWatchStarted = false let jobUplWatchStarted = false
global.db.onReady.then(() => { wiki.db.onReady.then(() => {
return global.db.Entry.remove({}) return wiki.db.Entry.remove({})
}).then(() => { }).then(() => {
job = new Cron({ job = new Cron({
cronTime: '0 */5 * * * *', cronTime: '0 */5 * * * *',
...@@ -84,17 +62,17 @@ global.db.onReady.then(() => { ...@@ -84,17 +62,17 @@ global.db.onReady.then(() => {
// Make sure we don't start two concurrent jobs // Make sure we don't start two concurrent jobs
if (jobIsBusy) { if (jobIsBusy) {
global.winston.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)') wiki.logger.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)')
return return
} }
global.winston.info('Running all jobs...') wiki.logger.info('Running all jobs...')
jobIsBusy = true jobIsBusy = true
// Prepare async job collector // Prepare async job collector
let jobs = [] let jobs = []
let repoPath = path.resolve(ROOTPATH, appconfig.paths.repo) let repoPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo)
let dataPath = path.resolve(ROOTPATH, appconfig.paths.data) let dataPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data)
let uploadsTempPath = path.join(dataPath, 'temp-upload') let uploadsTempPath = path.join(dataPath, 'temp-upload')
// ---------------------------------------- // ----------------------------------------
...@@ -105,7 +83,7 @@ global.db.onReady.then(() => { ...@@ -105,7 +83,7 @@ global.db.onReady.then(() => {
// -> Sync with Git remote // -> Sync with Git remote
//* **************************************** //* ****************************************
jobs.push(global.git.resync().then(() => { jobs.push(wiki.git.resync().then(() => {
// -> Stream all documents // -> Stream all documents
let cacheJobs = [] let cacheJobs = []
...@@ -185,18 +163,18 @@ global.db.onReady.then(() => { ...@@ -185,18 +163,18 @@ global.db.onReady.then(() => {
// ---------------------------------------- // ----------------------------------------
Promise.all(jobs).then(() => { Promise.all(jobs).then(() => {
global.winston.info('All jobs completed successfully! Going to sleep for now.') wiki.logger.info('All jobs completed successfully! Going to sleep for now.')
if (!jobUplWatchStarted) { if (!jobUplWatchStarted) {
jobUplWatchStarted = true jobUplWatchStarted = true
global.upl.initialScan().then(() => { wiki.upl.initialScan().then(() => {
job.start() job.start()
}) })
} }
return true return true
}).catch((err) => { }).catch((err) => {
global.winston.error('One or more jobs have failed: ', err) wiki.logger.error('One or more jobs have failed: ', err)
}).finally(() => { }).finally(() => {
jobIsBusy = false jobIsBusy = false
}) })
...@@ -212,7 +190,7 @@ global.db.onReady.then(() => { ...@@ -212,7 +190,7 @@ global.db.onReady.then(() => {
// ---------------------------------------- // ----------------------------------------
process.on('disconnect', () => { process.on('disconnect', () => {
global.winston.warn('Lost connection to main server. Exiting...') wiki.logger.warn('Lost connection to main server. Exiting...')
job.stop() job.stop()
process.exit() process.exit()
}) })
......
yarn.lock
View file @ 9c112ab5
This diff was suppressed by a .gitattributes entry.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment