fix: telemetry

CHANGELOG.md

@@ -12,6 +12,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Fixed
 - Fixed root admin refresh token fail
 - Fixed error page metadata title warning
+- Fixed telemetry
 
 ### Changed
 - Moved Insert Media button in Markdown editor

client/components/admin/admin-utilities.vue

@@ -95,7 +95,7 @@
                   v-card-text
                     v-subheader What is telemetry?
                     .body-1.pl-3 Telemetry allows the developers of Wiki.js to improve the software by collecting basic anonymized data about its usage and the host info. #[br] This is entirely optional and #[strong absolutely no] private data (such as content or personal data) is collected.
-                    .body-1.pt-3.pl-3 For maximum privacy, a random client ID is generated every time Wiki.js is started. This ID is used to group requests together while keeping complete anonymity.
+                    .body-1.pt-3.pl-3 For maximum privacy, a random client ID is generated during setup. This ID is used to group requests together while keeping complete anonymity. You can reset and generate a new one below at any time.
                     v-divider.my-3
                     v-subheader What is collected?
                     .body-1.pl-3 When telemetry is enabled, only the following data is transmitted:
@@ -105,13 +105,13 @@
                         v-list-tile-content: v-list-tile-title.caption Version of Wiki.js installed
                       v-list-tile
                         v-list-tile-avatar: v-icon info_outline
-                        v-list-tile-content: v-list-tile-title.caption Basic OS information (version, CPU cores, RAM available)
+                        v-list-tile-content: v-list-tile-title.caption Basic OS information (platform, CPU cores count, DB type)
                       v-list-tile
                         v-list-tile-avatar: v-icon info_outline
                         v-list-tile-content: v-list-tile-title.caption Crash debug data
                       v-list-tile
                         v-list-tile-avatar: v-icon info_outline
-                        v-list-tile-content: v-list-tile-title.caption Setup analytics (step reached)
+                        v-list-tile-content: v-list-tile-title.caption Setup analytics (installation checkpoint reached)
                     .body-2.pl-3
                     v-divider.my-3
                     v-subheader Settings

client/components/setup.vue

@@ -3,7 +3,7 @@
     v-toolbar(color='blue darken-2', dark, app, clipped-left, fixed, flat)
       v-spacer
       v-toolbar-title
-        span.subheading Wiki.js Setup
+        span.subheading Wiki.js {{wikiVersion}}
       v-spacer
     v-content.white
       v-progress-linear.ma-0(indeterminate, height='4', :active='loading')
@@ -193,9 +193,7 @@
               v-btn(color='success', @click='finish', v-if='loading || final.ok', :disabled='loading') Continue
 
     v-footer.pa-3(app, absolute, color='grey darken-3', height='auto')
-      .caption.grey--text Wiki.js
-      v-spacer
-      .caption.grey--text(v-if='conf.telemetry') Telemetry Client ID: {{telemetryId}}
+      .caption.grey--text Wiki.js {{wikiVersion}} Installation Wizard
 </template>
 
 <script>

package.json

@@ -39,15 +39,15 @@
     "node": ">=10.12"
   },
   "dependencies": {
+    "@bugsnag/js": "5.2.0",
     "apollo-fetch": "0.7.0",
-    "apollo-server": "2.3.2",
-    "apollo-server-express": "2.3.2",
+    "apollo-server": "2.3.3",
+    "apollo-server-express": "2.3.3",
     "auto-load": "3.0.4",
     "axios": "0.18.0",
     "bcryptjs-then": "1.0.1",
     "bluebird": "3.5.3",
     "body-parser": "1.18.3",
-    "bugsnag": "2.4.3",
     "bull": "3.6.0",
     "chalk": "2.4.2",
     "cheerio": "1.0.0-rc.2",
@@ -62,7 +62,7 @@
     "dependency-graph": "0.8.0",
     "diff": "4.0.1",
     "diff2html": "2.7.0",
-    "dotize": "^0.2.0",
+    "dotize": "^0.3.0",
     "execa": "1.0.0",
     "express": "4.16.4",
     "express-brute": "1.0.1",
@@ -77,7 +77,7 @@
     "graphql-list-fields": "2.0.2",
     "graphql-subscriptions": "1.0.0",
     "graphql-tools": "4.0.4",
-    "highlight.js": "9.13.1",
+    "highlight.js": "9.14.2",
     "i18next": "14.0.1",
     "i18next-express-middleware": "1.7.1",
     "i18next-localstorage-cache": "1.1.1",
@@ -157,13 +157,13 @@
     "simple-git": "1.107.0",
     "sqlite3": "4.0.6",
     "subscriptions-transport-ws": "0.9.15",
-    "twemoji": "11.2.0",
+    "twemoji": "11.3.0",
     "uslug": "1.0.4",
     "uuid": "3.3.2",
     "validate.js": "0.12.0",
     "validator": "10.11.0",
     "validator-as-promised": "1.0.2",
-    "winston": "3.1.0",
+    "winston": "3.2.1",
     "yargs": "12.0.5"
   },
   "devDependencies": {
@@ -184,14 +184,14 @@
     "animated-number-vue": "0.1.4",
     "apollo-cache-inmemory": "1.4.2",
     "apollo-client": "2.4.12",
-    "apollo-link": "1.2.6",
-    "apollo-link-batch-http": "1.2.6",
-    "apollo-link-error": "1.1.5",
-    "apollo-link-http": "1.5.9",
+    "apollo-link": "1.2.8",
+    "apollo-link-batch-http": "1.2.8",
+    "apollo-link-error": "1.1.7",
+    "apollo-link-http": "1.5.11",
     "apollo-link-persisted-queries": "0.2.2",
-    "apollo-link-ws": "1.0.12",
+    "apollo-link-ws": "1.0.14",
     "apollo-utilities": "1.1.2",
-    "autoprefixer": "9.4.6",
+    "autoprefixer": "9.4.7",
     "babel-eslint": "10.0.1",
     "babel-jest": "24.0.0",
     "babel-loader": "^8.0.5",
@@ -208,17 +208,17 @@
     "cssnano": "4.1.8",
     "duplicate-package-checker-webpack-plugin": "3.0.0",
     "epic-spinners": "1.0.4",
-    "eslint": "5.12.1",
+    "eslint": "5.13.0",
     "eslint-config-requarks": "1.0.7",
     "eslint-config-standard": "12.0.0",
-    "eslint-plugin-import": "2.15.0",
+    "eslint-plugin-import": "2.16.0",
     "eslint-plugin-node": "8.0.1",
     "eslint-plugin-promise": "4.0.1",
     "eslint-plugin-standard": "4.0.0",
     "eslint-plugin-vue": "5.1.0",
     "file-loader": "3.0.1",
     "filesize.js": "1.0.2",
-    "grapesjs": "0.14.50",
+    "grapesjs": "0.14.52",
     "graphiql": "0.12.0",
     "graphql-persisted-document-loader": "1.0.1",
     "graphql-tag": "^2.10.1",
@@ -256,7 +256,7 @@
     "stylus-loader": "3.0.2",
     "twemoji-awesome": "1.0.6",
     "url-loader": "1.1.2",
-    "vee-validate": "2.1.5",
+    "vee-validate": "2.1.7",
     "velocity-animate": "1.5.2",
     "viz.js": "2.1.2",
     "vue": "2.5.22",
@@ -265,7 +265,7 @@
     "vue-clipboards": "1.2.4",
     "vue-codemirror": "4.0.6",
     "vue-hot-reload-api": "2.3.1",
-    "vue-loader": "15.6.1",
+    "vue-loader": "15.6.2",
     "vue-material-design-icons": "2.6.0",
     "vue-moment": "4.0.0",
     "vue-router": "3.0.2",
@@ -275,8 +275,8 @@
     "vue-tour": "1.1.0",
     "vue2-animate": "2.1.0",
     "vuedraggable": "2.17.0",
-    "vuescroll": "4.9.6",
-    "vuetify": "1.4.3",
+    "vuescroll": "4.10.2",
+    "vuetify": "1.4.4",
     "vuex": "3.1.0",
     "vuex-pathify": "1.1.3",
     "vuex-persistedstate": "2.5.4",
@@ -286,10 +286,10 @@
     "webpack-dev-middleware": "3.5.1",
     "webpack-hot-middleware": "2.24.3",
     "webpack-merge": "4.2.1",
-    "webpack-subresource-integrity": "1.3.1",
+    "webpack-subresource-integrity": "1.3.2",
     "whatwg-fetch": "3.0.0",
     "write-file-webpack-plugin": "4.5.0",
-    "xterm": "3.10.1",
+    "xterm": "3.11.0",
     "zxcvbn": "4.4.2"
   },
   "browserslist": [

server/app/data.yml

@@ -86,5 +86,5 @@ telemetry:
   BUGSNAG_ID: 'bb4b324d0675bcbba10025617fd2cec8'
   BUGSNAG_REMOTE: 'https://notify.bugsnag.com'
   GA_ID: 'UA-9094100-7'
-  GA_REMOTE: 'https://www.google-analytics.com/collect'
+  GA_REMOTE: 'https://www.google-analytics.com/batch'
 # ---------------------------------

server/core/db.js

@@ -90,9 +90,6 @@ module.exports = {
     const models = autoload(path.join(WIKI.SERVERPATH, 'models'))
 
     // Set init tasks
-
-    console.info(migrationSource)
-
     let initTasks = {
       // -> Migrate DB Schemas
       async syncSchemas() {

server/core/kernel.js

@@ -40,6 +40,7 @@ module.exports = {
         WIKI.logger.info('Starting setup wizard...')
         require('../setup')()
       } else {
+        await this.initTelemetry()
         await require('../master')()
         this.postBootMaster()
       }
@@ -61,5 +62,20 @@ module.exports = {
 
     await WIKI.auth.activateStrategies()
     await WIKI.queue.start()
+  },
+  /**
+   * Init Telemetry
+   */
+  async initTelemetry() {
+    require('./telemetry').init()
+
+    process.on('unhandledRejection', (err) => {
+      WIKI.logger.warn(err)
+      WIKI.telemetry.sendError(err)
+    })
+    process.on('uncaughtException', (err) => {
+      WIKI.logger.warn(err)
+      WIKI.telemetry.sendError(err)
+    })
   }
 }

server/core/system.js

@@ -35,8 +35,6 @@ module.exports = {
    * @param {Object} opts Options object
    */
   async upgradeFromMongo (opts) {
-    WIKI.telemetry.sendEvent('setup', 'upgradeFromMongo')
-
     WIKI.logger.info('Upgrading from MongoDB...')
 
     let mongo = require('mongodb').MongoClient

server/core/telemetry.js

 const _ = require('lodash')
 const axios = require('axios')
-const bugsnag = require('bugsnag')
-const path = require('path')
+const bugsnag = require('@bugsnag/node')
 const uuid = require('uuid/v4')
+const qs = require('querystring')
+const os = require('os')
 
 /* global WIKI */
 
 module.exports = {
-  cid: '',
+  client: null,
   enabled: false,
   init() {
-    this.cid = uuid()
-    bugsnag.register(WIKI.data.telemetry.BUGSNAG_ID, {
+    this.client = bugsnag({
+      apiKey: WIKI.data.telemetry.BUGSNAG_ID,
+      appType: 'server',
       appVersion: WIKI.version,
       autoNotify: false,
-      hostname: this.cid,
+      collectUserIp: false,
+      hostname: _.get(WIKI.config, 'telemetry.clientId', uuid()),
       notifyReleaseStages: ['production'],
-      packageJSON: path.join(WIKI.ROOTPATH, 'package.json'),
+      releaseStage: WIKI.IS_DEBUG ? 'development' : 'production',
       projectRoot: WIKI.ROOTPATH,
-      useSSL: true
-    })
-    bugsnag.onBeforeNotify((notification, originalError) => {
-      if (!this.enabled) { return false }
+      logger: null,
+      beforeSend: (report) => {
+        if (!WIKI.telemetry.enabled) { return false }
+      }
     })
+    WIKI.telemetry = this
 
-    if (_.get(WIKI.config, 'logging.telemetry', false) === true) {
+    if (_.get(WIKI.config, 'telemetry.isEnabled', false) === true) {
       this.enabled = true
+      this.sendOSInfo()
     }
-
-    return this
+  },
+  sendOSInfo() {
+    this.sendBatchEvents([
+      {
+        eventCategory: 'node-version',
+        eventAction: process.version
+      },
+      {
+        eventCategory: 'os-platform',
+        eventAction: os.platform()
+      },
+      {
+        eventCategory: 'cpu-cores',
+        eventAction: os.cpus().length
+      },
+      {
+        eventCategory: 'db-type',
+        eventAction: WIKI.config.db.type
+      }
+    ])
   },
   sendError(err) {
-    bugsnag.notify(err, { userId: this.cid })
+    this.client.notify(err)
   },
   sendEvent(eventCategory, eventAction, eventLabel) {
-    if (!this.enabled) { return false }
+    this.sendBatchEvents([{
+      eventCategory,
+      eventAction,
+      eventLabel
+    }])
+  },
+  sendBatchEvents(events) {
+    if (!this.enabled || WIKI.IS_DEBUG) { return false }
     axios({
       method: 'post',
       url: WIKI.data.telemetry.GA_REMOTE,
       headers: {
-        'Content-type': 'application/x-www-form-urlencoded'
+        'Content-type': 'text/plain'
       },
-      params: {
+      data: events.map(ev => {
+        return qs.stringify({
           v: 1, // API version
           tid: WIKI.data.telemetry.GA_ID, // Tracking ID
           aip: 1, // Anonymize IP
           ds: 'server', // Data source
-        cid: this.cid, // Client ID
+          cid: WIKI.telemetry.cid, // Client ID
           t: 'event', // Hit Type
-        ec: eventCategory, // Event Category
-        ea: eventAction, // Event Action
-        el: eventLabel // Event Label
-      }
+          ec: ev.eventCategory, // Event Category
+          ea: ev.eventAction, // Event Action
+          el: ev.eventLabel // Event Label
+        })
+      }).join('\n')
     }).then(resp => {
       if (resp.status !== 200) {
         WIKI.logger.warn('Unable to send analytics telemetry request.')
@@ -59,5 +91,9 @@ module.exports = {
     }, err => {
       WIKI.logger.warn('Unable to send analytics telemetry request.')
     })
+  },
+  generateClientId() {
+    _.set(WIKI.config, 'telemetry.clientId', uuid())
+    return WIKI.config.telemetry.clientId
   }
 }

server/index.js

@@ -25,21 +25,6 @@ WIKI.configSvc.init()
 WIKI.logger = require('./core/logger').init('MASTER')
 
 // ----------------------------------------
-// Init Telemetry
-// ----------------------------------------
-
-WIKI.telemetry = require('./core/telemetry').init()
-
-process.on('unhandledRejection', (err) => {
-  WIKI.logger.warn(err)
-  WIKI.telemetry.sendError(err)
-})
-process.on('uncaughtException', (err) => {
-  WIKI.logger.warn(err)
-  WIKI.telemetry.sendError(err)
-})
-
-// ----------------------------------------
 // Start Kernel
 // ----------------------------------------
 

server/setup.js

@@ -71,38 +71,14 @@ module.exports = () => {
 
   app.get('*', async (req, res) => {
     let packageObj = await fs.readJson(path.join(WIKI.ROOTPATH, 'package.json'))
-    res.render('setup', {
-      packageObj,
-      telemetryClientID: WIKI.telemetry.cid
-    })
+    res.render('setup', { packageObj })
   })
 
   /**
    * Finalize
    */
   app.post('/finalize', async (req, res) => {
-    WIKI.telemetry.sendEvent('setup', 'finalize')
-
     try {
-      // Basic checks
-      if (!semver.satisfies(process.version, '>=10.14')) {
-        throw new Error('Node.js 10.14.x or later required!')
-      }
-
-      // Upgrade from WIKI.js 1.x?
-      if (req.body.upgrade) {
-        await WIKI.system.upgradeFromMongo({
-          mongoCnStr: cfgHelper.parseConfigValue(req.body.upgMongo)
-        })
-      }
-
-      // Create directory structure
-      WIKI.logger.info('Creating data directories...')
-      const dataPath = path.join(process.cwd(), 'data')
-      await fs.ensureDir(dataPath)
-      await fs.emptyDir(path.join(dataPath, 'cache'))
-      await fs.ensureDir(path.join(dataPath, 'uploads'))
-
       // Set config
       _.set(WIKI.config, 'auth', {
         audience: 'urn:wiki.js',
@@ -149,7 +125,7 @@ module.exports = () => {
       _.set(WIKI.config, 'sessionSecret', (await crypto.randomBytesAsync(32)).toString('hex'))
       _.set(WIKI.config, 'telemetry', {
         isEnabled: req.body.telemetry === true,
-        clientId: WIKI.telemetry.cid
+        clientId: uuid()
       })
       _.set(WIKI.config, 'theming', {
         theme: 'default',
@@ -157,6 +133,30 @@ module.exports = () => {
       })
       _.set(WIKI.config, 'title', 'Wiki.js')
 
+      // Init Telemetry
+      WIKI.kernel.initTelemetry()
+      WIKI.telemetry.sendEvent('setup', 'install-start')
+
+      // Basic checks
+      if (!semver.satisfies(process.version, '>=10.14')) {
+        throw new Error('Node.js 10.14.x or later required!')
+      }
+
+      // Upgrade from WIKI.js 1.x?
+      if (req.body.upgrade) {
+        WIKI.telemetry.sendEvent('setup', 'install-mongo-upgrade')
+        await WIKI.system.upgradeFromMongo({
+          mongoCnStr: cfgHelper.parseConfigValue(req.body.upgMongo)
+        })
+      }
+
+      // Create directory structure
+      WIKI.logger.info('Creating data directories...')
+      const dataPath = path.join(process.cwd(), 'data')
+      await fs.ensureDir(dataPath)
+      await fs.emptyDir(path.join(dataPath, 'cache'))
+      await fs.ensureDir(path.join(dataPath, 'uploads'))
+
       // Generate certificates
       WIKI.logger.info('Generating certificates...')
       const certs = crypto.generateKeyPairSync('rsa', {
@@ -244,6 +244,8 @@ module.exports = () => {
       await WIKI.models.searchEngines.refreshSearchEnginesFromDisk()
       await WIKI.models.searchEngines.query().patch({ isEnabled: true }).where('key', 'db')
 
+      WIKI.telemetry.sendEvent('setup', 'install-loadedmodules')
+
       // Load storage targets
       await WIKI.models.storage.refreshTargetsFromDisk()
 
@@ -305,6 +307,7 @@ module.exports = () => {
       })
 
       WIKI.logger.info('Setup is complete!')
+      WIKI.telemetry.sendEvent('setup', 'install-completed')
       res.json({
         ok: true,
         redirectPath: '/',
@@ -321,6 +324,7 @@ module.exports = () => {
         }, 1000)
       })
     } catch (err) {
+      WIKI.telemetry.sendError(err)
       res.json({ ok: false, error: err.message })
     }
   })