feat: storage modes + git improvements

client/components/admin/admin-groups-edit-permissions.vue

@@ -19,7 +19,7 @@
         ) This is a system group. Some permissions cannot be modified.
     v-container.px-3.pb-3.pt-0(fluid, grid-list-md)
       v-layout(row, wrap)
-        v-flex(xs12, md6, lg4, v-for='pmGroup in permissions')
+        v-flex(xs12, md6, lg4, v-for='pmGroup in permissions', :key='pmGroup.category')
           v-card.md2(flat, :class='$vuetify.dark ? "grey darken-3-d5" : "white"')
             v-subheader {{pmGroup.category}}
             v-card-text.pt-0

client/components/admin/admin-storage.vue

@@ -20,18 +20,33 @@
             v-tab(v-for='tgt in activeTargets', :key='tgt.key') {{ tgt.title }}
 
             v-tab-item(key='settings', :transition='false', :reverse-transition='false')
-              v-card.pa-3(flat, tile)
-                .body-2.grey--text.text--darken-1 Select which storage targets to enable:
-                .caption.grey--text.pb-2 Some storage targets require additional configuration in their dedicated tab (when selected).
-                v-form
-                  v-checkbox.my-0(
-                    v-for='tgt in targets'
-                    v-model='tgt.isEnabled'
-                    :key='tgt.key'
-                    :label='tgt.title'
-                    color='primary'
-                    hide-details
-                  )
+              v-container.pa-3(fluid, grid-list-md)
+                v-layout(row, wrap)
+                  v-flex(xs12, md6)
+                    .body-2.grey--text.text--darken-1 Select which storage targets to enable:
+                    .caption.grey--text.pb-2 Some storage targets require additional configuration in their dedicated tab (when selected).
+                    v-form
+                      v-checkbox.my-0(
+                        :disabled='!tgt.isAvailable'
+                        v-for='tgt in targets'
+                        v-model='tgt.isEnabled'
+                        :key='tgt.key'
+                        :label='tgt.title'
+                        color='primary'
+                        hide-details
+                      )
+                  v-flex(xs12, md6)
+                    .pa-3.grey.radius-7(:class='$vuetify.dark ? "darken-4" : "lighten-5"')
+                      .body-2.grey--text.text--darken-1 Advanced Settings
+                      v-text-field.mt-3.md2(
+                        v-model='syncInterval'
+                        outline
+                        background-color='grey lighten-2'
+                        prepend-icon='schedule'
+                        label='Synchronization Interval'
+                        hint='For performance reasons, some storage targets synchronize changes on an interval-based schedule, instead of on every change. Define at which interval should the synchronization occur for all storage targets.'
+                        persistent-hint
+                      )
 
             v-tab-item(v-for='(tgt, n) in activeTargets', :key='tgt.key', :transition='false', :reverse-transition='false')
               v-card.pa-3(flat, tile)
@@ -58,7 +73,7 @@
                       persistent-hint
                       :class='cfg.value.hint ? "mb-2" : ""'
                     )
-                    v-switch(
+                    v-switch.mb-3(
                       v-else-if='cfg.value.type === "boolean"'
                       :key='cfg.key'
                       :label='cfg.value.title'
@@ -89,23 +104,26 @@
                         label='Bi-directional'
                         color='primary'
                         value='sync'
+                        :disabled='tgt.supportedModes.indexOf(`sync`) < 0'
                       )
                       v-radio(
                         label='Push to target'
                         color='primary'
                         value='push'
+                        :disabled='tgt.supportedModes.indexOf(`push`) < 0'
                       )
                       v-radio(
                         label='Pull from target'
                         color='primary'
                         value='pull'
+                        :disabled='tgt.supportedModes.indexOf(`pull`) < 0'
                       )
                   .body-1.ml-3
-                    strong Bi-directional
+                    strong Bi-directional #[em.red--text.text--lighten-2(v-if='tgt.supportedModes.indexOf(`sync`) < 0') Unsupported]
                     .pb-3 In bi-directional mode, content is first pulled from the storage target. Any newer content overwrites local content. New content since last sync is then pushed to the storage target, overwriting any content on target if present.
-                    strong Push to target
-                    .pb-3 Content is always pushed to the storage target, overwriting any existing content. This is the default and safest choice for backup scenarios.
-                    strong Pull from target
+                    strong Push to target #[em.red--text.text--lighten-2(v-if='tgt.supportedModes.indexOf(`push`) < 0') Unsupported]
+                    .pb-3 Content is always pushed to the storage target, overwriting any existing content. This is safest choice for backup scenarios.
+                    strong Pull from target #[em.red--text.text--lighten-2(v-if='tgt.supportedModes.indexOf(`pull`) < 0') Unsupported]
                     .pb-3 Content is always pulled from the storage target, overwriting any local content which already exists. This choice is usually reserved for single-use content import. Caution with this option as any local content will always be overwritten!
 </template>
 
@@ -121,6 +139,7 @@ export default {
   },
   data() {
     return {
+      syncInterval: '5m',
       targets: []
     }
   },
@@ -163,7 +182,13 @@ export default {
     targets: {
       query: targetsQuery,
       fetchPolicy: 'network-only',
-      update: (data) => _.cloneDeep(data.storage.targets).map(str => ({...str, config: str.config.map(cfg => ({...cfg, value: JSON.parse(cfg.value)}))})),
+      update: (data) => _.cloneDeep(data.storage.targets).map(str => ({
+        ...str,
+        config: _.sortBy(str.config.map(cfg => ({
+          ...cfg,
+          value: JSON.parse(cfg.value)
+        })), [t => t.value.order])
+      })),
       watchLoading (isLoading) {
         this.$store.commit(`loading${isLoading ? 'Start' : 'Stop'}`, 'admin-storage-refresh')
       }

client/graph/admin/storage/storage-query-targets.gql

 query {
   storage {
     targets(orderBy: "title ASC") {
+      isAvailable
       isEnabled
       key
       title
       description
       logo
       website
+      supportedModes
       mode
       config {
         key

package.json

@@ -83,7 +83,7 @@
     "i18next-localstorage-cache": "1.1.1",
     "i18next-node-fs-backend": "2.1.1",
     "image-size": "0.7.1",
-    "ioredis": "4.5.1",
+    "ioredis": "4.6.2",
     "js-base64": "2.5.1",
     "js-binary": "1.2.0",
     "js-yaml": "3.12.1",
@@ -266,7 +266,7 @@
     "vue-codemirror": "4.0.6",
     "vue-hot-reload-api": "2.3.1",
     "vue-loader": "15.6.2",
-    "vue-material-design-icons": "2.6.0",
+    "vue-material-design-icons": "3.0.0",
     "vue-moment": "4.0.0",
     "vue-router": "3.0.2",
     "vue-simple-breakpoints": "1.0.3",

server/core/kernel.js

@@ -61,6 +61,7 @@ module.exports = {
     await WIKI.models.storage.refreshTargetsFromDisk()
 
     await WIKI.auth.activateStrategies()
+    await WIKI.models.storage.initTargets()
     await WIKI.queue.start()
   },
   /**

server/graph/schemas/storage.graphql

@@ -36,12 +36,14 @@ type StorageMutation {
 # -----------------------------------------------
 
 type StorageTarget {
+  isAvailable: Boolean!
   isEnabled: Boolean!
   key: String!
   title: String!
   description: String
   logo: String
   website: String
+  supportedModes: [String]
   mode: String
   config: [KeyValuePair]
 }

server/helpers/common.js

@@ -30,7 +30,8 @@ module.exports = {
         type: (value.type || value).toLowerCase(),
         title: value.title || _.startCase(key),
         hint: value.hint || false,
-        enum: value.enum || false
+        enum: value.enum || false,
+        order: value.order || 100
       })
       return result
     }, {})

server/models/pages.js

@@ -224,7 +224,9 @@ module.exports = class Page extends Model {
         'pages.*',
         {
           authorName: 'author.name',
-          creatorName: 'creator.name'
+          authorEmail: 'author.email',
+          creatorName: 'creator.name',
+          creatorEmail: 'creator.email'
         }
       ])
       .joinRelation('author')

server/models/storage.js

@@ -48,6 +48,7 @@ module.exports = class Storage extends Model {
       }
       WIKI.data.storage = diskTargets.map(target => ({
         ...target,
+        isAvailable: _.get(target, 'isAvailable', false),
         props: commonHelper.parseModuleProps(target.props)
       }))
 
@@ -58,7 +59,7 @@ module.exports = class Storage extends Model {
           newTargets.push({
             key: target.key,
             isEnabled: false,
-            mode: 'push',
+            mode: target.defaultMode || 'push',
             config: _.transform(target.props, (result, value, key) => {
               _.set(result, key, value.default)
               return result
@@ -100,10 +101,9 @@ module.exports = class Storage extends Model {
     try {
       for(let target of targets) {
         target.fn = require(`../modules/storage/${target.key}/storage`)
-        await target.fn.init.call({
-          config: target.config,
-          mode: target.mode
-        })
+        target.fn.config = target.config
+        target.fn.mode = target.mode
+        await target.fn.init()
       }
     } catch (err) {
       WIKI.logger.warn(err)
@@ -114,11 +114,7 @@ module.exports = class Storage extends Model {
   static async pageEvent({ event, page }) {
     try {
       for(let target of targets) {
-        await target.fn[event].call({
-          config: target.config,
-          mode: target.mode,
-          page
-        })
+        await target.fn[event](page)
       }
     } catch (err) {
       WIKI.logger.warn(err)

server/modules/storage/disk/definition.yml

@@ -4,10 +4,19 @@ description: Local storage on disk or network shares.
 author: requarks.io
 logo: https://static.requarks.io/logo/local-fs.svg
 website: https://wiki.js.org
+isAvailable: true
 supportedModes:
   - push
+defaultMode: push
 props:
   path:
     type: String
     title: Path
     hint: Absolute path without a trailing slash (e.g. /home/wiki/backup, C:\wiki\backup)
+    order: 1
+  createDailyBackups:
+    type: Boolean
+    default: false
+    title: Create Daily Backups
+    hint: A tar.gz archive containing all content will be created daily in subfolder named _daily. Archives are kept for a month.
+    order: 2

server/modules/storage/disk/storage.js

@@ -46,23 +46,32 @@ module.exports = {
     // not used
   },
   async init() {
+    WIKI.logger.info('(STORAGE/DISK) Initializing...')
     await fs.ensureDir(this.config.path)
+    WIKI.logger.info('(STORAGE/DISK) Initialization completed.')
   },
-  async created() {
-    const filePath = path.join(this.config.path, `${this.page.path}.${getFileExtension(this.page.contentType)}`)
-    await fs.outputFile(filePath, injectMetadata(this.page), 'utf8')
+  async sync() {
+    // not used
+  },
+  async created(page) {
+    WIKI.logger.info(`(STORAGE/DISK) Creating file ${page.path}...`)
+    const filePath = path.join(this.config.path, `${page.path}.${getFileExtension(page.contentType)}`)
+    await fs.outputFile(filePath, injectMetadata(page), 'utf8')
   },
-  async updated() {
-    const filePath = path.join(this.config.path, `${this.page.path}.${getFileExtension(this.page.contentType)}`)
-    await fs.outputFile(filePath, injectMetadata(this.page), 'utf8')
+  async updated(page) {
+    WIKI.logger.info(`(STORAGE/DISK) Updating file ${page.path}...`)
+    const filePath = path.join(this.config.path, `${page.path}.${getFileExtension(page.contentType)}`)
+    await fs.outputFile(filePath, injectMetadata(page), 'utf8')
   },
-  async deleted() {
-    const filePath = path.join(this.config.path, `${this.page.path}.${getFileExtension(this.page.contentType)}`)
+  async deleted(page) {
+    WIKI.logger.info(`(STORAGE/DISK) Deleting file ${page.path}...`)
+    const filePath = path.join(this.config.path, `${page.path}.${getFileExtension(page.contentType)}`)
     await fs.unlink(filePath)
   },
-  async renamed() {
-    const sourceFilePath = path.join(this.config.path, `${this.page.sourcePath}.${getFileExtension(this.page.contentType)}`)
-    const destinationFilePath = path.join(this.config.path, `${this.page.destinationPath}.${getFileExtension(this.page.contentType)}`)
+  async renamed(page) {
+    WIKI.logger.info(`(STORAGE/DISK) Renaming file ${page.sourcePath} to ${page.destinationPath}...`)
+    const sourceFilePath = path.join(this.config.path, `${page.sourcePath}.${getFileExtension(page.contentType)}`)
+    const destinationFilePath = path.join(this.config.path, `${page.destinationPath}.${getFileExtension(page.contentType)}`)
     await fs.move(sourceFilePath, destinationFilePath, { overwrite: true })
   }
 }

server/modules/storage/git/definition.yml

@@ -4,10 +4,12 @@ description: Git is a version control system for tracking changes in computer fi
 author: requarks.io
 logo: https://static.requarks.io/logo/git-alt.svg
 website: https://git-scm.com/
+isAvailable: true
 supportedModes:
   - sync
   - push
   - pull
+defaultMode: sync
 props:
   authType:
     type: String
@@ -17,42 +19,52 @@ props:
     enum:
       - 'basic'
       - 'ssh'
+    order: 1
   repoUrl:
     type: String
     title: Repository URI
     hint: Git-compliant URI (e.g. git@github.com:org/repo.git for ssh, https://github.com/org/repo.git for basic)
+    order: 2
   branch:
     type: String
     default: 'master'
+    order: 3
   verifySSL:
     type: Boolean
     default: true
     title: Verify SSL Certificate
     hint: Some hosts requires SSL certificate checking to be disabled. Leave enabled for proper security.
+    order: 31
   sshPrivateKeyPath:
     type: String
     title: SSH Private Key Path
     hint: SSH Authentication Only - Absolute path to the key. The key must NOT be passphrase-protected.
+    order: 10
   basicUsername:
     type: String
     title: Username
     hint: Basic Authentication Only
+    order: 11
   basicPassword:
     type: String
     title: Password / PAT
     hint: Basic Authentication Only
+    order: 12
   localRepoPath:
     type: String
     title: Local Repository Path
     default: './data/repo'
     hint: 'Path where the local git repository will be created.'
+    order: 30
   defaultEmail:
     type: String
     title: Default Author Email
     default: 'name@company.com'
     hint: 'Used as fallback in case the author of the change is not present.'
+    order: 20
   defaultName:
     type: String
     title: Default Author Name
     default: 'John Smith'
     hint: 'Used as fallback in case the author of the change is not present.'
+    order: 21

server/modules/storage/git/storage.js

@@ -3,8 +3,6 @@ const sgit = require('simple-git/promise')
 const fs = require('fs-extra')
 const _ = require('lodash')
 
-let repoPath = path.join(process.cwd(), 'data/repo')
-
 /**
  * Get file extension based on content type
  */
@@ -43,72 +41,80 @@ const injectMetadata = (page) => {
 }
 
 module.exports = {
+  git: null,
+  repoPath: path.join(process.cwd(), 'data/repo'),
   async activated() {
-
+    // not used
   },
   async deactivated() {
-
+    // not used
   },
   async init() {
     WIKI.logger.info('(STORAGE/GIT) Initializing...')
-    repoPath = path.resolve(WIKI.ROOTPATH, this.config.localRepoPath)
-    await fs.ensureDir(repoPath)
-    const git = sgit(repoPath)
+    this.repoPath = path.resolve(WIKI.ROOTPATH, this.config.localRepoPath)
+    await fs.ensureDir(this.repoPath)
+    this.git = sgit(this.repoPath)
 
     // Initialize repo (if needed)
     WIKI.logger.info('(STORAGE/GIT) Checking repository state...')
-    const isRepo = await git.checkIsRepo()
+    const isRepo = await this.git.checkIsRepo()
     if (!isRepo) {
       WIKI.logger.info('(STORAGE/GIT) Initializing local repository...')
-      await git.init()
+      await this.git.init()
     }
 
     // Set default author
-    await git.raw(['config', '--local', 'user.email', this.config.defaultEmail])
-    await git.raw(['config', '--local', 'user.name', this.config.defaultName])
+    await this.git.raw(['config', '--local', 'user.email', this.config.defaultEmail])
+    await this.git.raw(['config', '--local', 'user.name', this.config.defaultName])
 
     // Purge existing remotes
     WIKI.logger.info('(STORAGE/GIT) Listing existing remotes...')
-    const remotes = await git.getRemotes()
+    const remotes = await this.git.getRemotes()
     if (remotes.length > 0) {
       WIKI.logger.info('(STORAGE/GIT) Purging existing remotes...')
       for(let remote of remotes) {
-        await git.removeRemote(remote.name)
+        await this.git.removeRemote(remote.name)
       }
     }
 
     // Add remote
     WIKI.logger.info('(STORAGE/GIT) Setting SSL Verification config...')
-    await git.raw(['config', '--local', '--bool', 'http.sslVerify', _.toString(this.config.verifySSL)])
+    await this.git.raw(['config', '--local', '--bool', 'http.sslVerify', _.toString(this.config.verifySSL)])
     switch (this.config.authType) {
       case 'ssh':
         WIKI.logger.info('(STORAGE/GIT) Setting SSH Command config...')
-        await git.addConfig('core.sshCommand', `ssh -i "${this.config.sshPrivateKeyPath}" -o StrictHostKeyChecking=no`)
+        await this.git.addConfig('core.sshCommand', `ssh -i "${this.config.sshPrivateKeyPath}" -o StrictHostKeyChecking=no`)
         WIKI.logger.info('(STORAGE/GIT) Adding origin remote via SSH...')
-        await git.addRemote('origin', this.config.repoUrl)
+        await this.git.addRemote('origin', this.config.repoUrl)
         break
       default:
         WIKI.logger.info('(STORAGE/GIT) Adding origin remote via HTTPS...')
-        await git.addRemote('origin', `https://${this.config.basicUsername}:${this.config.basicPassword}@${this.config.repoUrl}`)
+        await this.git.addRemote('origin', `https://${this.config.basicUsername}:${this.config.basicPassword}@${this.config.repoUrl}`)
         break
     }
 
     // Fetch updates for remote
     WIKI.logger.info('(STORAGE/GIT) Fetch updates from remote...')
-    await git.raw(['remote', 'update', 'origin'])
+    await this.git.raw(['remote', 'update', 'origin'])
 
     // Checkout branch
-    const branches = await git.branch()
+    const branches = await this.git.branch()
     if (!_.includes(branches.all, this.config.branch) && !_.includes(branches.all, `remotes/origin/${this.config.branch}`)) {
       throw new Error('Invalid branch! Make sure it exists on the remote first.')
     }
     WIKI.logger.info(`(STORAGE/GIT) Checking out branch ${this.config.branch}...`)
-    await git.checkout(this.config.branch)
+    await this.git.checkout(this.config.branch)
+
+    // Perform initial sync
+    await this.sync()
 
+    WIKI.logger.info('(STORAGE/GIT) Initialization completed.')
+  },
+  async sync() {
     // Pull rebase
     if (_.includes(['sync', 'pull'], this.mode)) {
       WIKI.logger.info(`(STORAGE/GIT) Performing pull rebase from origin on branch ${this.config.branch}...`)
-      await git.pull('origin', this.config.branch, ['--rebase'])
+      await this.git.pull('origin', this.config.branch, ['--rebase'])
     }
 
     // Push
@@ -118,50 +124,48 @@ module.exports = {
       if (this.mode === 'push') {
         pushOpts.push('--force')
       }
-      await git.push('origin', this.config.branch, pushOpts)
+      await this.git.push('origin', this.config.branch, pushOpts)
     }
-
-    WIKI.logger.info('(STORAGE/GIT) Initialization completed.')
   },
-  async created() {
-    const fileName = `${this.page.path}.${getFileExtension(this.page.contentType)}`
-    const filePath = path.join(repoPath, fileName)
-    await fs.outputFile(filePath, injectMetadata(this.page), 'utf8')
-
-    const git = sgit(repoPath)
-    await git.add(`./${fileName}`)
-    await git.commit(`docs: create ${this.page.path}`, fileName, {
-      '--author': `"${this.page.authorName} <${this.page.authorEmail}>"`
+  async created(page) {
+    WIKI.logger.info(`(STORAGE/GIT) Committing new file ${page.path}...`)
+    const fileName = `${page.path}.${getFileExtension(page.contentType)}`
+    const filePath = path.join(this.repoPath, fileName)
+    await fs.outputFile(filePath, injectMetadata(page), 'utf8')
+
+    await this.git.add(`./${fileName}`)
+    await this.git.commit(`docs: create ${page.path}`, fileName, {
+      '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
   },
-  async updated() {
-    const fileName = `${this.page.path}.${getFileExtension(this.page.contentType)}`
-    const filePath = path.join(repoPath, fileName)
-    await fs.outputFile(filePath, injectMetadata(this.page), 'utf8')
-
-    const git = sgit(repoPath)
-    await git.add(`./${fileName}`)
-    await git.commit(`docs: update ${this.page.path}`, fileName, {
-      '--author': `"${this.page.authorName} <${this.page.authorEmail}>"`
+  async updated(page) {
+    WIKI.logger.info(`(STORAGE/GIT) Committing updated file ${page.path}...`)
+    const fileName = `${page.path}.${getFileExtension(page.contentType)}`
+    const filePath = path.join(this.repoPath, fileName)
+    await fs.outputFile(filePath, injectMetadata(page), 'utf8')
+
+    await this.git.add(`./${fileName}`)
+    await this.git.commit(`docs: update ${page.path}`, fileName, {
+      '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
   },
-  async deleted() {
-    const fileName = `${this.page.path}.${getFileExtension(this.page.contentType)}`
+  async deleted(page) {
+    WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${page.path}...`)
+    const fileName = `${page.path}.${getFileExtension(page.contentType)}`
 
-    const git = sgit(repoPath)
-    await git.rm(`./${fileName}`)
-    await git.commit(`docs: delete ${this.page.path}`, fileName, {
-      '--author': `"${this.page.authorName} <${this.page.authorEmail}>"`
+    await this.git.rm(`./${fileName}`)
+    await this.git.commit(`docs: delete ${page.path}`, fileName, {
+      '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
   },
-  async renamed() {
-    const sourceFilePath = `${this.page.sourcePath}.${getFileExtension(this.page.contentType)}`
-    const destinationFilePath = `${this.page.destinationPath}.${getFileExtension(this.page.contentType)}`
-
-    const git = sgit(repoPath)
-    await git.mv(`./${sourceFilePath}`, `./${destinationFilePath}`)
-    await git.commit(`docs: rename ${this.page.sourcePath} to ${destinationFilePath}`, destinationFilePath, {
-      '--author': `"${this.page.authorName} <${this.page.authorEmail}>"`
+  async renamed(page) {
+    WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${page.sourcePath} to ${page.destinationPath}...`)
+    const sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}`
+    const destinationFilePath = `${page.destinationPath}.${getFileExtension(page.contentType)}`
+
+    await this.git.mv(`./${sourceFilePath}`, `./${destinationFilePath}`)
+    await this.git.commit(`docs: rename ${page.sourcePath} to ${destinationFilePath}`, destinationFilePath, {
+      '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
   }
 }