feat: scheduler + admin page

package.json

@@ -64,6 +64,7 @@
     "connect-session-knex": "3.0.0",
     "cookie-parser": "1.4.6",
     "cors": "2.8.5",
+    "cron-parser": "4.6.0",
     "cuint": "0.2.2",
     "custom-error-instance": "2.1.2",
     "dependency-graph": "0.9.0",

server/app/data.yml

@@ -80,17 +80,17 @@ defaults:
 jobs:
   purgeUploads:
     onInit: true
-    schedule: PT15M
+    schedule: '*/15 * * * *'
     offlineSkip: false
     repeat: true
   syncGraphLocales:
     onInit: true
-    schedule: P1D
+    schedule: '0 0 * * *'
     offlineSkip: true
     repeat: true
   syncGraphUpdates:
     onInit: true
-    schedule: P1D
+    schedule: '0 0 * * *'
     offlineSkip: true
     repeat: true
   rebuildTree:

server/controllers/ssl.js

@@ -26,13 +26,13 @@ router.get('/.well-known/acme-challenge/:token', (req, res, next) => {
 /**
  * Redirect to HTTPS if HTTP Redirection is enabled
  */
-router.all('/*', (req, res, next) => {
-  if (WIKI.config.server.sslRedir && !req.secure && WIKI.servers.servers.https) {
-    let query = (!_.isEmpty(req.query)) ? `?${qs.stringify(req.query)}` : ``
-    return res.redirect(`https://${req.hostname}${req.originalUrl}${query}`)
-  } else {
-    next()
-  }
-})
+// router.all('/*', (req, res, next) => {
+//   if (WIKI.config.server.sslRedir && !req.secure && WIKI.servers.servers.https) {
+//     let query = (!_.isEmpty(req.query)) ? `?${qs.stringify(req.query)}` : ``
+//     return res.redirect(`https://${req.hostname}${req.originalUrl}${query}`)
+//   } else {
+//     next()
+//   }
+// })
 
 module.exports = router

server/core/kernel.js

@@ -76,7 +76,8 @@ module.exports = {
     await WIKI.models.commentProviders.initProvider()
     await WIKI.models.sites.reloadCache()
     await WIKI.models.storage.initTargets()
-    WIKI.scheduler.start()
+    await WIKI.scheduler.start()
+    await WIKI.scheduler.registerScheduledJobs()
 
     await WIKI.models.subscribeToNotifications()
   },

server/core/scheduler.js

@@ -6,10 +6,11 @@ const os = require('node:os')
 
 module.exports = {
   pool: null,
-  scheduler: null,
+  boss: null,
+  maxWorkers: 1,
   async init () {
     WIKI.logger.info('Initializing Scheduler...')
-    this.scheduler = new PgBoss({
+    this.boss = new PgBoss({
       db: {
         close: () => Promise.resolve('ok'),
         executeSql: async (text, values) => {
@@ -27,12 +28,14 @@ module.exports = {
       // ...WIKI.models.knex.client.connectionSettings,
       application_name: 'Wiki.js Scheduler',
       schema: WIKI.config.db.schemas.scheduler,
-      uuid: 'v4'
+      uuid: 'v4',
+      archiveCompletedAfterSeconds: 120,
+      deleteAfterHours: 24
     })
 
-    const maxWorkers = WIKI.config.workers === 'auto' ? os.cpus().length : WIKI.config.workers
-    WIKI.logger.info(`Initializing Worker Pool (Max ${maxWorkers})...`)
-    this.pool = new DynamicThreadPool(1, maxWorkers, './server/worker.js', {
+    this.maxWorkers = WIKI.config.workers === 'auto' ? os.cpus().length : WIKI.config.workers
+    WIKI.logger.info(`Initializing Worker Pool (Limit: ${this.maxWorkers})...`)
+    this.pool = new DynamicThreadPool(1, this.maxWorkers, './server/worker.js', {
       errorHandler: (err) => WIKI.logger.warn(err),
       exitHandler: () => WIKI.logger.debug('A worker has gone offline.'),
       onlineHandler: () => WIKI.logger.debug('New worker is online.')
@@ -41,20 +44,40 @@ module.exports = {
   },
   async start () {
     WIKI.logger.info('Starting Scheduler...')
-    await this.scheduler.start()
-    this.scheduler.work('*', async job => {
-      return this.pool.execute({
-        id: job.id,
-        name: job.name,
-        data: job.data
-      })
+    await this.boss.start()
+    this.boss.work('wk-*', {
+      teamSize: this.maxWorkers,
+      teamConcurrency: this.maxWorkers
+    }, async job => {
+      WIKI.logger.debug(`Starting job ${job.name}:${job.id}...`)
+      try {
+        const result = await this.pool.execute({
+          id: job.id,
+          name: job.name,
+          data: job.data
+        })
+        WIKI.logger.debug(`Completed job ${job.name}:${job.id}.`)
+        job.done(null, result)
+      } catch (err) {
+        WIKI.logger.warn(`Failed job ${job.name}:${job.id}): ${err.message}`)
+        job.done(err)
+      }
+      this.boss.complete(job.id)
     })
     WIKI.logger.info('Scheduler: [ STARTED ]')
   },
   async stop () {
     WIKI.logger.info('Stopping Scheduler...')
-    await this.scheduler.stop()
+    await this.boss.stop({ timeout: 5000 })
     await this.pool.destroy()
     WIKI.logger.info('Scheduler: [ STOPPED ]')
+  },
+  async registerScheduledJobs () {
+    for (const [key, job] of Object.entries(WIKI.data.jobs)) {
+      if (job.schedule) {
+        WIKI.logger.debug(`Scheduling regular job ${key}...`)
+        await this.boss.schedule(`wk-${key}`, job.schedule)
+      }
+    }
   }
 }

server/graph/resolvers/system.js

@@ -7,6 +7,7 @@ const path = require('path')
 const fs = require('fs-extra')
 const { DateTime } = require('luxon')
 const graphHelper = require('../../helpers/graph')
+const cronParser = require('cron-parser')
 
 /* global WIKI */
 
@@ -27,6 +28,35 @@ module.exports = {
     },
     systemSecurity () {
       return WIKI.config.security
+    },
+    async systemJobs (obj, args) {
+      switch (args.type) {
+        case 'ACTIVE': {
+          // const result = await WIKI.scheduler.boss.fetch('*', 25, { includeMeta: true })
+          return []
+        }
+        case 'COMPLETED': {
+          const result = await WIKI.scheduler.boss.fetchCompleted('*', 25, { includeMeta: true })
+          console.info(result)
+          return result ?? []
+        }
+        default: {
+          WIKI.logger.warn('Invalid Job Type requested.')
+          return []
+        }
+      }
+    },
+    async systemScheduledJobs (obj, args) {
+      const jobs = await WIKI.scheduler.boss.getSchedules()
+      return jobs.map(job => ({
+        id: job.name,
+        name: job.name,
+        cron: job.cron,
+        timezone: job.timezone,
+        nextExecution: cronParser.parseExpression(job.cron, { tz: job.timezone }).next(),
+        createdAt: job.created_on,
+        updatedAt: job.updated_on
+      }))
     }
   },
   Mutation: {

server/graph/schemas/system.graphql

@@ -7,6 +7,10 @@ extend type Query {
   systemFlags: [SystemFlag]
   systemInfo: SystemInfo
   systemSecurity: SystemSecurity
+  systemJobs(
+    type: SystemJobType!
+  ): [SystemJob]
+  systemScheduledJobs: [SystemScheduledJob]
 }
 
 extend type Mutation {
@@ -144,3 +148,25 @@ enum SystemSecurityCorsMode {
   HOSTNAMES
   REGEX
 }
+
+type SystemJob {
+  id: UUID
+  name: String
+  priority: Int
+  state: String
+}
+
+type SystemScheduledJob {
+  id: String
+  name: String
+  cron: String
+  timezone: String
+  nextExecution: Date
+  createdAt: Date
+  updatedAt: Date
+}
+
+enum SystemJobType {
+  ACTIVE
+  COMPLETED
+}

server/models/locales.js

@@ -39,25 +39,26 @@ module.exports = class Locale extends Model {
   }
 
   static async getNavLocales({ cache = false } = {}) {
-    if (!WIKI.config.lang.namespacing) {
-      return []
-    }
+    return []
+    // if (!WIKI.config.lang.namespacing) {
+    //   return []
+    // }
 
-    if (cache) {
-      const navLocalesCached = await WIKI.cache.get('nav:locales')
-      if (navLocalesCached) {
-        return navLocalesCached
-      }
-    }
-    const navLocales = await WIKI.models.locales.query().select('code', 'nativeName AS name').whereIn('code', WIKI.config.lang.namespaces).orderBy('code')
-    if (navLocales) {
-      if (cache) {
-        await WIKI.cache.set('nav:locales', navLocales, 300)
-      }
-      return navLocales
-    } else {
-      WIKI.logger.warn('Site Locales for navigation are missing or corrupted.')
-      return []
-    }
+    // if (cache) {
+    //   const navLocalesCached = await WIKI.cache.get('nav:locales')
+    //   if (navLocalesCached) {
+    //     return navLocalesCached
+    //   }
+    // }
+    // const navLocales = await WIKI.models.locales.query().select('code', 'nativeName AS name').whereIn('code', WIKI.config.lang.namespaces).orderBy('code')
+    // if (navLocales) {
+    //   if (cache) {
+    //     await WIKI.cache.set('nav:locales', navLocales, 300)
+    //   }
+    //   return navLocales
+    // } else {
+    //   WIKI.logger.warn('Site Locales for navigation are missing or corrupted.')
+    //   return []
+    // }
   }
 }

ux/public/_assets/icons/fluent-bot.svg

+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 48 48" width="96px" height="96px"><defs><linearGradient id="p0leOTPLvuNkjL_fSa~qVa" x1="24" x2="24" y1="9.109" y2="13.568" data-name="Безымянный градиент 6" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#0077d2"/><stop offset="1" stop-color="#0b59a2"/></linearGradient><linearGradient id="p0leOTPLvuNkjL_fSa~qVb" x1="4.5" x2="4.5" y1="26.717" y2="41.786" xlink:href="#p0leOTPLvuNkjL_fSa~qVa"/><linearGradient id="p0leOTPLvuNkjL_fSa~qVc" x1="43.5" x2="43.5" y1="26.717" y2="41.786" xlink:href="#p0leOTPLvuNkjL_fSa~qVa"/><linearGradient id="p0leOTPLvuNkjL_fSa~qVd" x1="16" x2="16" y1="25.054" y2="43.495" xlink:href="#p0leOTPLvuNkjL_fSa~qVa"/><linearGradient id="p0leOTPLvuNkjL_fSa~qVe" x1="32" x2="32" y1="25.054" y2="43.495" xlink:href="#p0leOTPLvuNkjL_fSa~qVa"/></defs><rect width="2" height="6" x="23" y="8" fill="url(#p0leOTPLvuNkjL_fSa~qVa)"/><path fill="url(#p0leOTPLvuNkjL_fSa~qVb)" d="M6,27H8a0,0,0,0,1,0,0V37a0,0,0,0,1,0,0H6a5,5,0,0,1-5-5v0A5,5,0,0,1,6,27Z"/><path fill="url(#p0leOTPLvuNkjL_fSa~qVc)" d="M40,27h2a5,5,0,0,1,5,5v0a5,5,0,0,1-5,5H40a0,0,0,0,1,0,0V27A0,0,0,0,1,40,27Z"/><path fill="#199be2" d="M24,13h0A18,18,0,0,1,42,31v8a2,2,0,0,1-2,2H8a2,2,0,0,1-2-2V31A18,18,0,0,1,24,13Z"/><circle cx="16" cy="31" r="6" fill="url(#p0leOTPLvuNkjL_fSa~qVd)"/><circle cx="32" cy="31" r="6" fill="url(#p0leOTPLvuNkjL_fSa~qVe)"/><circle cx="32" cy="31" r="4" fill="#50e6ff"/><circle cx="32" cy="31" r="2" fill="url(#p0leOTPLvuNkjL_fSa~qVe)"/><circle cx="16" cy="31" r="4" fill="#50e6ff"/><circle cx="16" cy="31" r="2" fill="url(#p0leOTPLvuNkjL_fSa~qVd)"/><circle cx="24" cy="8" r="2" fill="#199be2"/></svg>
\ No newline at end of file

ux/src/i18n/locales/en.json

@@ -1505,5 +1505,17 @@
   "profile.saveFailed": "Failed to save profile changes.",
   "admin.users.pronouns": "Pronouns",
   "admin.users.pronounsHint": "The pronouns used to address this user.",
-  "admin.users.appearance": "Site Appearance"
+  "admin.users.appearance": "Site Appearance",
+  "admin.scheduler.title": "Scheduler",
+  "admin.scheduler.subtitle": "View scheduled and completed jobs",
+  "admin.scheduler.active": "Active",
+  "admin.scheduler.completed": "Completed",
+  "admin.scheduler.scheduled": "Scheduled",
+  "admin.scheduler.activeNone": "There are no active jobs at the moment.",
+  "admin.scheduler.completedNone": "There are no recently completed job to display.",
+  "admin.scheduler.scheduledNone": "There are no scheduled jobs at the moment.",
+  "admin.scheduler.cron": "Cron",
+  "admin.scheduler.nextExecutionIn": "Next run {date}",
+  "admin.scheduler.nextExecution": "Next Run",
+  "admin.scheduler.timezone": "Timezone"
 }

ux/src/layouts/AdminLayout.vue

@@ -152,6 +152,10 @@ q-layout.admin(view='hHh Lpr lff')
           q-item-section {{ t('admin.mail.title') }}
           q-item-section(side)
             status-light(:color='adminStore.info.isMailConfigured ? `positive` : `warning`')
+        q-item(to='/_admin/scheduler', v-ripple, active-class='bg-primary text-white')
+          q-item-section(avatar)
+            q-icon(name='img:/_assets/icons/fluent-bot.svg')
+          q-item-section {{ t('admin.scheduler.title') }}
         q-item(to='/_admin/security', v-ripple, active-class='bg-primary text-white')
           q-item-section(avatar)
             q-icon(name='img:/_assets/icons/fluent-protect.svg')

ux/src/pages/AdminScheduler.vue

+<template lang='pug'>
+q-page.admin-terminal
+  .row.q-pa-md.items-center
+    .col-auto
+      img.admin-icon.animated.fadeInLeft(src='/_assets/icons/fluent-bot.svg')
+    .col.q-pl-md
+      .text-h5.text-primary.animated.fadeInLeft {{ t('admin.scheduler.title') }}
+      .text-subtitle1.text-grey.animated.fadeInLeft.wait-p2s {{ t('admin.scheduler.subtitle') }}
+    .col-auto.flex
+      q-btn-toggle.q-mr-md(
+        v-model='state.displayMode'
+        push
+        no-caps
+        :disable='state.loading > 0'
+        :toggle-color='$q.dark.isActive ? `white` : `black`'
+        :toggle-text-color='$q.dark.isActive ? `black` : `white`'
+        :text-color='$q.dark.isActive ? `white` : `black`'
+        :color='$q.dark.isActive ? `dark-1` : `white`'
+        :options=`[
+          { label: t('admin.scheduler.scheduled'), value: 'scheduled' },
+          { label: t('admin.scheduler.completed'), value: 'completed' }
+        ]`
+      )
+      q-separator.q-mr-md(vertical)
+      q-btn.q-mr-sm.acrylic-btn(
+        icon='las la-question-circle'
+        flat
+        color='grey'
+        :href='siteStore.docsBase + `/admin/scheduler`'
+        target='_blank'
+        type='a'
+        )
+      q-btn.q-mr-sm.acrylic-btn(
+        icon='las la-redo-alt'
+        flat
+        color='secondary'
+        :loading='state.loading > 0'
+        @click='load'
+      )
+  q-separator(inset)
+  .q-pa-md.q-gutter-md
+    template(v-if='state.displayMode === `scheduled`')
+      q-card.rounded-borders(
+        v-if='state.scheduledJobs.length < 1'
+        flat
+        :class='$q.dark.isActive ? `bg-dark-5 text-white` : `bg-grey-3 text-dark`'
+        )
+        q-card-section.items-center(horizontal)
+          q-card-section.col-auto.q-pr-none
+            q-icon(name='las la-info-circle', size='sm')
+          q-card-section.text-caption {{ t('admin.scheduler.scheduledNone') }}
+      q-card.shadow-1(v-else)
+        q-table(
+          :rows='state.scheduledJobs'
+          :columns='scheduledJobsHeaders'
+          row-key='name'
+          flat
+          hide-bottom
+          :rows-per-page-options='[0]'
+          :loading='state.loading > 0'
+          )
+          template(v-slot:body-cell-id='props')
+            q-td(:props='props')
+              q-spinner-clock.q-mr-sm(
+                color='indigo'
+                size='xs'
+              )
+              //- q-icon(name='las la-stopwatch', color='primary', size='sm')
+          template(v-slot:body-cell-name='props')
+            q-td(:props='props')
+              strong {{props.value}}
+          template(v-slot:body-cell-cron='props')
+            q-td(:props='props')
+              span {{ props.value }}
+          template(v-slot:body-cell-date='props')
+            q-td(:props='props')
+              i18n-t.text-caption(keypath='admin.scheduler.nextExecutionIn', tag='div')
+                template(#date)
+                  strong {{ humanizeDate(props.value) }}
+              small {{props.value}}
+    template(v-else)
+      q-card.rounded-borders(
+        v-if='state.jobs.length < 1'
+        flat
+        :class='$q.dark.isActive ? `bg-dark-5 text-white` : `bg-grey-3 text-dark`'
+        )
+        q-card-section.items-center(horizontal)
+          q-card-section.col-auto.q-pr-none
+            q-icon(name='las la-info-circle', size='sm')
+          q-card-section.text-caption {{ t('admin.scheduler.completedNone') }}
+      q-card.shadow-1(v-else) ---
+
+</template>
+
+<script setup>
+import { onMounted, reactive, watch } from 'vue'
+import { useMeta, useQuasar } from 'quasar'
+import { useI18n } from 'vue-i18n'
+import gql from 'graphql-tag'
+import { DateTime } from 'luxon'
+
+import { useSiteStore } from 'src/stores/site'
+
+// QUASAR
+
+const $q = useQuasar()
+
+// STORES
+
+const siteStore = useSiteStore()
+
+// I18N
+
+const { t } = useI18n()
+
+// META
+
+useMeta({
+  title: t('admin.scheduler.title')
+})
+
+// DATA
+
+const state = reactive({
+  displayMode: 'scheduled',
+  scheduledJobs: [],
+  jobs: [],
+  loading: 0
+})
+
+const scheduledJobsHeaders = [
+  {
+    align: 'center',
+    field: 'id',
+    name: 'id',
+    sortable: false,
+    style: 'width: 15px; padding-right: 0;'
+  },
+  {
+    label: t('common.field.name'),
+    align: 'left',
+    field: 'name',
+    name: 'name',
+    sortable: true
+  },
+  {
+    label: t('admin.scheduler.cron'),
+    align: 'left',
+    field: 'cron',
+    name: 'cron',
+    sortable: false
+  },
+  {
+    label: t('admin.scheduler.timezone'),
+    align: 'left',
+    field: 'timezone',
+    name: 'timezone',
+    sortable: false
+  },
+  {
+    label: t('admin.scheduler.nextExecution'),
+    align: 'left',
+    field: 'nextExecution',
+    name: 'date',
+    sortable: false
+  }
+]
+
+// WATCHERS
+
+watch(() => state.displayMode, (newValue) => {
+  load()
+})
+
+// METHODS
+
+async function load () {
+  state.loading++
+  try {
+    if (state.displayMode === 'scheduled') {
+      const resp = await APOLLO_CLIENT.query({
+        query: gql`
+          query getSystemScheduledJobs {
+            systemScheduledJobs {
+              id
+              name
+              cron
+              timezone
+              nextExecution
+            }
+          }
+        `,
+        fetchPolicy: 'network-only'
+      })
+      state.scheduledJobs = resp?.data?.systemScheduledJobs
+    } else {
+      const resp = await APOLLO_CLIENT.query({
+        query: gql`
+          query getSystemJobs (
+            $type: SystemJobType!
+          ) {
+            systemJobs (
+              type: $type
+              ) {
+              id
+              name
+              priority
+              state
+            }
+          }
+        `,
+        variables: {
+          type: state.displayMode.toUpperCase()
+        },
+        fetchPolicy: 'network-only'
+      })
+      state.jobs = resp?.data?.systemJobs
+    }
+  } catch (err) {
+    $q.notify({
+      type: 'negative',
+      message: 'Failed to load scheduled jobs.',
+      caption: err.message
+    })
+  }
+  state.loading--
+}
+
+function humanizeDate (val) {
+  return DateTime.fromISO(val).toRelative()
+}
+
+// MOUNTED
+
+onMounted(() => {
+  load()
+})
+</script>
+
+<style lang='scss'>
+.admin-terminal {
+  &-term {
+    width: 100%;
+    background-color: #000;
+    border-radius: 5px;
+    overflow: hidden;
+    padding: 10px;
+  }
+}
+</style>

ux/src/router/routes.js

@@ -47,6 +47,7 @@ const routes = [
       { path: 'api', component: () => import('pages/AdminApi.vue') },
       { path: 'extensions', component: () => import('pages/AdminExtensions.vue') },
       { path: 'mail', component: () => import('pages/AdminMail.vue') },
+      { path: 'scheduler', component: () => import('pages/AdminScheduler.vue') },
       { path: 'security', component: () => import('pages/AdminSecurity.vue') },
       { path: 'system', component: () => import('pages/AdminSystem.vue') },
       { path: 'terminal', component: () => import('pages/AdminTerminal.vue') },