Skip to content

W
wiki-js
You need to sign in or sign up before continuing.
Commit 1d289376 authored by NGPixel's avatar NGPixel
Browse files
Options

Git pull & push functionnality + caching work

parent cc1e6b44
Hide whitespace changes
Inline Side-by-side
Showing with 225 additions and 245 deletions
config.sample.yml
View file @ 1d289376
...@@ -26,8 +26,8 @@ port: 80 ...@@ -26,8 +26,8 @@ port: 80
# ------------------------------------------------- # -------------------------------------------------
datadir: datadir:
repo: ./repo
db: ./data db: ./data
uploads: ./uploads
# ------------------------------------------------- # -------------------------------------------------
# Git Connection Info # Git Connection Info
...@@ -35,23 +35,22 @@ datadir: ...@@ -35,23 +35,22 @@ datadir:
# Full explanation + examples in the documentation (https://requarks-wiki.readme.io/) # Full explanation + examples in the documentation (https://requarks-wiki.readme.io/)
git: git:
path: ./repo
remote: true
url: https://github.com/Organization/Repo url: https://github.com/Organization/Repo
branch: master branch: master
auth: auth:
# Type: basic, oauth or ssh
type: ssh type: ssh
user: gitusername
publickey: /etc/requarkswiki/keys/git.pub username: marty
privatekey: /etc/requarkswiki/keys/git.key
passphrase: SomeSshPassphrase # Password, OAuth token or private key passphrase:
# auth: password: MartyMcFly88
# type: oauth
# token: 1234567890abcdefghijklmnopqrstuvxyz # Only for SSH authentication:
# auth: publicKey: /etc/requarkswiki/keys/git.pub
# type: basic privateKey: /etc/requarkswiki/keys/git.key
# user: johnsmith sslVerify: true
# pass: password123
# ------------------------------------------------- # -------------------------------------------------
# Secret key to use when encrypting sessions # Secret key to use when encrypting sessions
......
controllers/pages.js
View file @ 1d289376
...@@ -3,21 +3,31 @@ ...@@ -3,21 +3,31 @@
var express = require('express'); var express = require('express');
var router = express.Router(); var router = express.Router();
router.get('/edit/*', (req, res, next) => {
res.send('EDIT MODE');
});
router.get('/new/*', (req, res, next) => {
res.send('CREATE MODE');
});
/** /**
* Home * Home
*/ */
router.get('/', (req, res) => { router.get('/*', (req, res, next) => {
var Promise = require('bluebird'); let safePath = entries.parsePath(req.path);
var fs = Promise.promisifyAll(require("fs"));
fs.readFileAsync("repo/Storage/Redis.md", "utf8").then(function(contents) { entries.fetch(safePath).then((pageData) => {
let pageData = mark.parse(contents); console.log(pageData);
if(!pageData.meta.title) { if(pageData) {
pageData.meta.title = 'Redis.md'; res.render('pages/view', { pageData });
} else {
next();
} }
res.render('pages/view', { pageData }); }).catch((err) => {
}); next();
});
}); });
......
gulpfile.js
View file @ 1d289376
...@@ -47,8 +47,7 @@ var paths = { ...@@ -47,8 +47,7 @@ var paths = {
], ],
fonts: [ fonts: [
'./node_modules/font-awesome/fonts/*-webfont.*', './node_modules/font-awesome/fonts/*-webfont.*',
'.!/node_modules/font-awesome/fonts/*-webfont.svg', '!./node_modules/font-awesome/fonts/*-webfont.svg'
'./node_modules/roboto-fontface/fonts/Roboto/*.woff'
], ],
deploypackage: [ deploypackage: [
'./**/*', './**/*',
...@@ -67,7 +66,7 @@ var paths = { ...@@ -67,7 +66,7 @@ var paths = {
gulp.task('server', ['scripts', 'css', 'fonts'], function() { gulp.task('server', ['scripts', 'css', 'fonts'], function() {
nodemon({ nodemon({
script: './server', script: './server',
ignore: ['assets/', 'client/', 'tests/'], ignore: ['assets/', 'client/', 'data/', 'repo/', 'tests/'],
ext: 'js json', ext: 'js json',
env: { 'NODE_ENV': 'development' } env: { 'NODE_ENV': 'development' }
}); });
......
models/loki.js → models/db.js
View file @ 1d289376
...@@ -6,6 +6,8 @@ var loki = require('lokijs'), ...@@ -6,6 +6,8 @@ var loki = require('lokijs'),
Promise = require('bluebird'), Promise = require('bluebird'),
_ = require('lodash'); _ = require('lodash');
var cols = ['User','Entry'];
/** /**
* Loki.js module * Loki.js module
* *
...@@ -27,27 +29,17 @@ module.exports = function(appconfig) { ...@@ -27,27 +29,17 @@ module.exports = function(appconfig) {
autosave: true, autosave: true,
autosaveInterval: 5000 autosaveInterval: 5000
}), }),
Models: {},
onReady: dbReady onReady: dbReady
}; };
// Load Models // Load Models
let dbModelsPath = path.join(ROOTPATH, 'models/db');
dbModel.Store.loadDatabase({}, () => { dbModel.Store.loadDatabase({}, () => {
fs _.forEach(cols, (col) => {
.readdirSync(dbModelsPath) dbModel[col] = dbModel.Store.getCollection(col);
.filter(function(file) { if(!dbModel[col]) {
return (file.indexOf(".") !== 0); dbModel[col] = dbModel.Store.addCollection(col);
})
.forEach(function(file) {
let modelName = _.upperFirst(_.split(file,'.')[0]);
dbModel.Models[modelName] = require(path.join(dbModelsPath, file));
dbModel[modelName] = dbModel.Store.getCollection(modelName);
if(!dbModel[modelName]) {
dbModel[modelName] = dbModel.Store.addCollection(modelName);
} }
}); });
......
models/db/user.js deleted 100644 → 0
View file @ cc1e6b44
"use strict";
var bcrypt = require('bcryptjs-then');
var Promise = require('bluebird');
var _ = require('lodash');
module.exports = {
};
\ No newline at end of file
models/entries.js 0 → 100644
View file @ 1d289376
"use strict";
var Promise = require('bluebird'),
path = require('path'),
fs = Promise.promisifyAll(require("fs")),
_ = require('lodash'),
farmhash = require('farmhash'),
msgpack = require('msgpack5')();
/**
* Entries Model
*/
module.exports = {
_repoPath: 'repo',
_cachePath: 'data/cache',
/**
* Initialize Entries model
*
* @param {Object} appconfig The application config
* @return {Object} Entries model instance
*/
init(appconfig) {
let self = this;
self._repoPath = appconfig.datadir.repo;
self._cachePath = path.join(appconfig.datadir.db, 'cache');
return self;
},
fetch(entryPath) {
let self = this;
let fpath = path.join(self._repoPath, entryPath + '.md');
let cpath = path.join(self._cachePath, farmhash.fingerprint32(entryPath) + '.bin');
return fs.statAsync(cpath).then((st) => {
return st.isFile();
}).catch((err) => {
return false;
}).then((isCache) => {
if(isCache) {
console.log('from cache!');
return fs.readFileAsync(cpath, 'utf8').then((contents) => {
return msgpack.decode(contents);
}).catch((err) => {
winston.error('Corrupted cache file. Deleting it...');
fs.unlinkSync(cpath);
return false;
});
} else {
console.log('original!');
// Parse original and cache it
return fs.statAsync(fpath).then((st) => {
if(st.isFile()) {
return fs.readFileAsync(fpath, 'utf8').then((contents) => {
let pageData = mark.parse(contents);
if(!pageData.meta.title) {
pageData.meta.title = entryPath;
}
let cacheData = msgpack.encode(pageData);
return fs.writeFileAsync(cpath, cacheData, { encoding: 'utf8' }).then(() => {
return pageData;
}).catch((err) => {
winston.error('Unable to write to cache! Performance may be affected.');
return pageData;
});
});
} else {
return false;
}
});
}
});
},
parsePath(urlPath) {
let wlist = new RegExp('[^a-z0-9/\-]','g');
urlPath = _.toLower(urlPath).replace(wlist, '');
if(urlPath === '/') {
urlPath = 'home';
}
let urlParts = _.filter(_.split(urlPath, '/'), (p) => { return !_.isEmpty(p); });
return _.join(urlParts, '/');
}
};
\ No newline at end of file
models/git.js
View file @ 1d289376
"use strict"; "use strict";
var NodeGit = require("nodegit"), var Git = require("git-wrapper2-promise"),
Promise = require('bluebird'), Promise = require('bluebird'),
path = require('path'), path = require('path'),
os = require('os'), os = require('os'),
fs = Promise.promisifyAll(require("fs")), fs = Promise.promisifyAll(require("fs")),
moment = require('moment'), moment = require('moment'),
_ = require('lodash'); _ = require('lodash'),
URL = require('url');
/** /**
* Git Model * Git Model
...@@ -14,11 +15,11 @@ var NodeGit = require("nodegit"), ...@@ -14,11 +15,11 @@ var NodeGit = require("nodegit"),
module.exports = { module.exports = {
_git: null, _git: null,
_url: '',
_repo: { _repo: {
path: '', path: '',
branch: 'master', branch: 'master',
exists: false, exists: false,
inst: null,
sync: true sync: true
}, },
_signature: { _signature: {
...@@ -42,16 +43,15 @@ module.exports = { ...@@ -42,16 +43,15 @@ module.exports = {
//-> Build repository path //-> Build repository path
if(_.isEmpty(appconfig.git.path) || appconfig.git.path === 'auto') { if(_.isEmpty(appconfig.datadir.repo)) {
self._repo.path = path.join(ROOTPATH, 'repo'); self._repo.path = path.join(ROOTPATH, 'repo');
} else { } else {
self._repo.path = appconfig.git.path; self._repo.path = appconfig.datadir.repo;
} }
//-> Initialize repository //-> Initialize repository
self._initRepo(appconfig).then((repo) => { self._initRepo(appconfig).then((repo) => {
self._repo.inst = repo;
if(self._repo.sync) { if(self._repo.sync) {
self.resync(); self.resync();
...@@ -61,8 +61,8 @@ module.exports = { ...@@ -61,8 +61,8 @@ module.exports = {
// Define signature // Define signature
self._signature.name = appconfig.git.userinfo.name || 'Wiki'; self._signature.name = appconfig.git.signature.name || 'Wiki';
self._signature.email = appconfig.git.userinfo.email || 'user@example.com'; self._signature.email = appconfig.git.signature.email || 'user@example.com';
return self; return self;
...@@ -78,7 +78,7 @@ module.exports = { ...@@ -78,7 +78,7 @@ module.exports = {
let self = this; let self = this;
winston.info('[GIT] Initializing Git repository...'); winston.info('[GIT] Checking Git repository...');
//-> Check if path is accessible //-> Check if path is accessible
...@@ -88,209 +88,94 @@ module.exports = { ...@@ -88,209 +88,94 @@ module.exports = {
} }
}).then(() => { }).then(() => {
self._git = new Git({ 'git-dir': self._repo.path });
//-> Check if path already contains a git working folder //-> Check if path already contains a git working folder
return fs.statAsync(path.join(self._repo.path, '.git')).then((stat) => { return self._git.isRepo().then((isRepo) => {
self._repo.exists = stat.isDirectory(); self._repo.exists = isRepo;
return (!isRepo) ? self._git.exec('init') : true;
}).catch((err) => { }).catch((err) => {
self._repo.exists = false; self._repo.exists = false;
}); });
}).then(() => { }).then(() => {
//-> Init repository // Initialize remote
let repoInitOperation = null; let urlObj = URL.parse(appconfig.git.url);
self._repo.branch = appconfig.git.branch; urlObj.auth = appconfig.git.auth.username + ((appconfig.git.auth.type !== 'ssh') ? ':' + appconfig.git.auth.password : '');
self._repo.sync = appconfig.git.remote; self._url = URL.format(urlObj);
self._opts.clone = self._generateCloneOptions(appconfig);
self._opts.push = self._generatePushOptions(appconfig); return self._git.exec('remote', 'show').then((cProc) => {
let out = cProc.stdout.toString();
if(self._repo.exists) { if(_.includes(out, 'origin')) {
return true;
winston.info('[GIT] Using existing repository...'); } else {
repoInitOperation = NodeGit.Repository.open(self._repo.path); return Promise.join(
self._git.exec('config', ['--local', 'user.name', self._signature.name]),
} else if(appconfig.git.remote) { self._git.exec('config', ['--local', 'user.email', self._signature.email])
).then(() => {
winston.info('[GIT] Cloning remote repository for first time...'); return self._git.exec('remote', ['add', 'origin', self._url]);
repoInitOperation = NodeGit.Clone(appconfig.git.url, self._repo.path, self._opts.clone); })
}
} else { });
winston.info('[GIT] Using offline local repository...');
repoInitOperation = NodeGit.Repository.init(self._repo.path, 0);
}
return repoInitOperation;
}).catch((err) => { }).catch((err) => {
winston.error('Unable to open or clone Git repository!'); winston.error('Git remote error!');
winston.error(err); throw err;
}).then((repo) => { }).then(() => {
if(self._repo.sync) {
NodeGit.Remote.setPushurl(repo, 'origin', appconfig.git.url);
}
return repo;
winston.info('[GIT] Git repository is now ready.'); winston.info('[GIT] Git repository is now ready.');
return true;
}); });
}, },
/**
* Generate Clone Options object
*
* @param {Object} appconfig The application configuration
* @return {Object} CloneOptions object
*/
_generateCloneOptions(appconfig) {
let cloneOptions = new NodeGit.CloneOptions();
cloneOptions.fetchOpts = this._generateFetchOptions(appconfig);
return cloneOptions;
},
_generateFetchOptions(appconfig) {
let fetchOptions = new NodeGit.FetchOptions();
fetchOptions.callbacks = this._generateRemoteCallbacks(appconfig);
return fetchOptions;
},
_generatePushOptions(appconfig) {
let pushOptions = new NodeGit.PushOptions();
pushOptions.callbacks = this._generateRemoteCallbacks(appconfig);
return pushOptions;
},
_generateRemoteCallbacks(appconfig) {
let remoteCallbacks = new NodeGit.RemoteCallbacks();
let credFunc = this._generateCredentials(appconfig);
remoteCallbacks.credentials = () => { return credFunc; };
remoteCallbacks.transferProgress = _.noop;
if(os.type() === 'Darwin') {
remoteCallbacks.certificateCheck = () => { return 1; }; // Bug in OS X, bypass certs check workaround
} else {
remoteCallbacks.certificateCheck = _.noop;
}
return remoteCallbacks;
},
_generateCredentials(appconfig) {
let cred = null;
switch(appconfig.git.auth.type) {
case 'basic':
cred = NodeGit.Cred.userpassPlaintextNew(
appconfig.git.auth.user,
appconfig.git.auth.pass
);
break;
case 'oauth':
cred = NodeGit.Cred.userpassPlaintextNew(
appconfig.git.auth.token,
"x-oauth-basic"
);
break;
case 'ssh':
cred = NodeGit.Cred.sshKeyNew(
appconfig.git.auth.user,
appconfig.git.auth.publickey,
appconfig.git.auth.privatekey,
appconfig.git.auth.passphrase
);
break;
default:
cred = NodeGit.Cred.defaultNew();
break;
}
return cred;
},
resync() { resync() {
let self = this; let self = this;
// Fetch // Fetch
return self._repo.inst.fetch('origin', self._opts.clone.fetchOpts) winston.info('[GIT] Performing pull from remote repository...');
.catch((err) => { return self._git.pull('origin', self._repo.branch).then((cProc) => {
winston.error('Unable to fetch from git origin!' + err); winston.info('[GIT] Pull completed.');
})
// Merge
.then(() => {
return self._repo.inst.mergeBranches(self._repo.branch, 'origin/' + self._repo.branch);
}) })
.catch((err) => { .catch((err) => {
winston.error('Unable to merge from remote head!' + err); winston.error('Unable to fetch from git origin!');
throw err;
}) })
// Push
.then(() => { .then(() => {
return self._repo.inst.getRemote('origin').then((remote) => {
// Get modified files
return self._repo.inst.refreshIndex().then((index) => {
return self._repo.inst.getStatus().then(function(arrayStatusFile) {
let addOp = [];
// Add to next commit
_.forEach(arrayStatusFile, (v) => {
addOp.push(arrayStatusFile[0].path());
});
console.log('DUDE1');
// Create Commit
let sig = NodeGit.Signature.create(self._signature.name, self._signature.email, moment().utc().unix(), 0);
return self._repo.inst.createCommitOnHead(addOp, sig, sig, "Wiki Sync").then(() => {
console.log('DUDE2'); // Check for changes
return remote.connect(NodeGit.Enums.DIRECTION.PUSH, self._opts.push.callbacks).then(() => { return self._git.exec('status').then((cProc) => {
let out = cProc.stdout.toString();
if(!_.includes(out, 'nothing to commit')) {
console.log('DUDE3'); // Add, commit and push
// Push to remote winston.info('[GIT] Performing push to remote repository...');
return self._git.add('-A').then(() => {
return remote.push( ["refs/heads/master:refs/heads/master"], self._opts.push).then((errNum) => { return self._git.commit("Resync");
console.log('DUDE' + errNum); }).then(() => {
}).catch((err) => { return self._git.push('origin', self._repo.branch);
console.log(err); }).then(() => {
}); return winston.info('[GIT] Push completed.');
});
});
}); } else {
winston.info('[GIT] Repository is already up to date. Nothing to commit.');
}
}); return true;
})
/**/
}); });
}).catch((err) => {
winston.error('Unable to push to git origin!' + err); })
.catch((err) => {
winston.error('Unable to push changes to remote!');
throw err;
}); });
} }
......
models/localdata.js
View file @ 1d289376
"use strict"; "use strict";
var fs = require('fs'), var fs = require('fs'),
path = require('path'),
_ = require('lodash'); _ = require('lodash');
/** /**
...@@ -10,7 +11,7 @@ var fs = require('fs'), ...@@ -10,7 +11,7 @@ var fs = require('fs'),
*/ */
module.exports = (appconfig) => { module.exports = (appconfig) => {
// Create DB folder // Create data directories
try { try {
fs.mkdirSync(appconfig.datadir.db); fs.mkdirSync(appconfig.datadir.db);
...@@ -21,10 +22,8 @@ module.exports = (appconfig) => { ...@@ -21,10 +22,8 @@ module.exports = (appconfig) => {
} }
} }
// Create Uploads folder
try { try {
fs.mkdirSync(appconfig.datadir.uploads); fs.mkdirSync(path.join(appconfig.datadir.db, 'cache'));
} catch (err) { } catch (err) {
if(err.code !== 'EEXIST') { if(err.code !== 'EEXIST') {
winston.error(err); winston.error(err);
......
package.json
View file @ 1d289376
...@@ -36,6 +36,7 @@ ...@@ -36,6 +36,7 @@
"body-parser": "^1.15.2", "body-parser": "^1.15.2",
"bulma": "^0.1.2", "bulma": "^0.1.2",
"cheerio": "^0.22.0", "cheerio": "^0.22.0",
"child-process-promise": "^2.1.3",
"compression": "^1.6.2", "compression": "^1.6.2",
"connect-flash": "^0.1.1", "connect-flash": "^0.1.1",
"connect-loki": "^1.0.6", "connect-loki": "^1.0.6",
...@@ -46,6 +47,8 @@ ...@@ -46,6 +47,8 @@
"express-brute-loki": "^1.0.0", "express-brute-loki": "^1.0.0",
"express-session": "^1.14.0", "express-session": "^1.14.0",
"express-validator": "^2.20.8", "express-validator": "^2.20.8",
"farmhash": "^1.2.0",
"git-wrapper2-promise": "^0.2.9",
"highlight.js": "^9.6.0", "highlight.js": "^9.6.0",
"i18next": "^3.4.1", "i18next": "^3.4.1",
"i18next-express-middleware": "^1.0.1", "i18next-express-middleware": "^1.0.1",
...@@ -66,11 +69,10 @@ ...@@ -66,11 +69,10 @@
"markdown-it-toc-and-anchor": "^4.1.1", "markdown-it-toc-and-anchor": "^4.1.1",
"moment": "^2.14.1", "moment": "^2.14.1",
"moment-timezone": "^0.5.5", "moment-timezone": "^0.5.5",
"nodegit": "^0.14.1", "msgpack5": "^3.4.0",
"passport": "^0.3.2", "passport": "^0.3.2",
"passport-local": "^1.0.0", "passport-local": "^1.0.0",
"pug": "^2.0.0-beta5", "pug": "^2.0.0-beta5",
"roboto-fontface": "^0.6.0",
"serve-favicon": "^2.3.0", "serve-favicon": "^2.3.0",
"simplemde": "^1.11.2", "simplemde": "^1.11.2",
"slug": "^0.9.1", "slug": "^0.9.1",
......
server.js
View file @ 1d289376
...@@ -4,14 +4,6 @@ ...@@ -4,14 +4,6 @@
// Licensed under AGPLv3 // Licensed under AGPLv3
// =========================================== // ===========================================
process.on('uncaughtException', function (exception) {
console.log(exception);
});
process.on('unhandledRejection', (reason, p) => {
console.log("Unhandled Rejection at: Promise ", p, " reason: ", reason);
// application specific logging, throwing an error, or other logic here
});
global.ROOTPATH = __dirname; global.ROOTPATH = __dirname;
// ---------------------------------------- // ----------------------------------------
...@@ -22,10 +14,11 @@ global.winston = require('winston'); ...@@ -22,10 +14,11 @@ global.winston = require('winston');
winston.info('[SERVER] Requarks Wiki is initializing...'); winston.info('[SERVER] Requarks Wiki is initializing...');
var appconfig = require('./models/config')('./config.yml'); var appconfig = require('./models/config')('./config.yml');
var lcdata = require('./models/localdata')(appconfig); let lcdata = require('./models/localdata');
global.db = require('./models/loki')(appconfig); global.db = require('./models/db')(appconfig);
global.git = require('./models/git').init(appconfig); global.git = require('./models/git').init(appconfig);
global.entries = require('./models/entries').init(appconfig);
global.mark = require('./models/markdown'); global.mark = require('./models/markdown');
// ---------------------------------------- // ----------------------------------------
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment