Unverified Commit c87f4ce7 authored by NGPixel's avatar NGPixel

feat: rebuild search index + isSearchable flag

parent 17034040
...@@ -12,6 +12,7 @@ export async function up (knex) { ...@@ -12,6 +12,7 @@ export async function up (knex) {
// ===================================== // =====================================
await knex.raw('CREATE EXTENSION IF NOT EXISTS ltree;') await knex.raw('CREATE EXTENSION IF NOT EXISTS ltree;')
await knex.raw('CREATE EXTENSION IF NOT EXISTS pgcrypto;') await knex.raw('CREATE EXTENSION IF NOT EXISTS pgcrypto;')
await knex.raw('CREATE EXTENSION IF NOT EXISTS pg_trgm;')
await knex.schema await knex.schema
// ===================================== // =====================================
...@@ -235,6 +236,8 @@ export async function up (knex) { ...@@ -235,6 +236,8 @@ export async function up (knex) {
table.string('editor').notNullable() table.string('editor').notNullable()
table.string('contentType').notNullable() table.string('contentType').notNullable()
table.boolean('isBrowsable').notNullable().defaultTo(true) table.boolean('isBrowsable').notNullable().defaultTo(true)
table.boolean('isSearchable').notNullable().defaultTo(true)
table.specificType('isSearchableComputed', `boolean GENERATED ALWAYS AS ("publishState" != 'draft' AND "isSearchable") STORED`).index()
table.string('password') table.string('password')
table.integer('ratingScore').notNullable().defaultTo(0) table.integer('ratingScore').notNullable().defaultTo(0)
table.integer('ratingCount').notNullable().defaultTo(0) table.integer('ratingCount').notNullable().defaultTo(0)
...@@ -393,6 +396,13 @@ export async function up (knex) { ...@@ -393,6 +396,13 @@ export async function up (knex) {
.table('userKeys', table => { .table('userKeys', table => {
table.uuid('userId').notNullable().references('id').inTable('users') table.uuid('userId').notNullable().references('id').inTable('users')
}) })
// =====================================
// TS WORD SUGGESTION TABLE
// =====================================
.createTable('autocomplete', table => {
table.text('word')
})
.raw(`CREATE INDEX "autocomplete_idx" ON "autocomplete" USING GIN (word gin_trgm_ops)`)
// ===================================== // =====================================
// DEFAULT DATA // DEFAULT DATA
...@@ -518,8 +528,8 @@ export async function up (knex) { ...@@ -518,8 +528,8 @@ export async function up (knex) {
key: 'update', key: 'update',
value: { value: {
lastCheckedAt: null, lastCheckedAt: null,
version: null, version: WIKI.version,
versionDate: null versionDate: WIKI.releaseDate
} }
}, },
{ {
...@@ -772,6 +782,11 @@ export async function up (knex) { ...@@ -772,6 +782,11 @@ export async function up (knex) {
type: 'system' type: 'system'
}, },
{ {
task: 'refreshAutocomplete',
cron: '0 */3 * * *',
type: 'system'
},
{
task: 'updateLocales', task: 'updateLocales',
cron: '0 0 * * *', cron: '0 0 * * *',
type: 'system' type: 'system'
......
...@@ -77,6 +77,7 @@ export default { ...@@ -77,6 +77,7 @@ export default {
.select(searchCols) .select(searchCols)
.fromRaw('pages, websearch_to_tsquery(?, ?) query', [dictName, args.query]) .fromRaw('pages, websearch_to_tsquery(?, ?) query', [dictName, args.query])
.where('siteId', args.siteId) .where('siteId', args.siteId)
.where('isSearchableComputed', true)
.where(builder => { .where(builder => {
if (args.path) { if (args.path) {
builder.where('path', 'ILIKE', `${args.path}%`) builder.where('path', 'ILIKE', `${args.path}%`)
......
import { generateError, generateSuccess } from '../../helpers/graph.mjs'
export default {
Mutation: {
async rebuildSearchIndex (obj, args, context) {
try {
await WIKI.data.searchEngine.rebuild()
return {
responseResult: generateSuccess('Index rebuilt successfully')
}
} catch (err) {
return generateError(err)
}
}
}
}
...@@ -141,6 +141,19 @@ export default { ...@@ -141,6 +141,19 @@ export default {
return generateError(err) return generateError(err)
} }
}, },
async rebuildSearchIndex (obj, args, context) {
try {
await WIKI.scheduler.addJob({
task: 'rebuildSearchIndex',
maxRetries: 0
})
return {
operation: generateSuccess('Search index rebuild has been scheduled and will start shortly.')
}
} catch (err) {
return generateError(err)
}
},
async retryJob (obj, args, context) { async retryJob (obj, args, context) {
WIKI.logger.info(`Admin requested rescheduling of job ${args.id}...`) WIKI.logger.info(`Admin requested rescheduling of job ${args.id}...`)
try { try {
......
...@@ -65,6 +65,11 @@ extend type Query { ...@@ -65,6 +65,11 @@ extend type Query {
limit: Int limit: Int
): PageSearchResponse ): PageSearchResponse
searchPagesAutocomplete(
siteId: UUID!
query: String!
): [String]
pages( pages(
limit: Int limit: Int
orderBy: PageOrderBy orderBy: PageOrderBy
...@@ -130,6 +135,7 @@ extend type Mutation { ...@@ -130,6 +135,7 @@ extend type Mutation {
editor: String! editor: String!
icon: String icon: String
isBrowsable: Boolean isBrowsable: Boolean
isSearchable: Boolean
locale: String! locale: String!
path: String! path: String!
publishState: PagePublishState! publishState: PagePublishState!
...@@ -231,6 +237,7 @@ type Page { ...@@ -231,6 +237,7 @@ type Page {
icon: String icon: String
id: UUID id: UUID
isBrowsable: Boolean isBrowsable: Boolean
isSearchable: Boolean
locale: String locale: String
password: String password: String
path: String path: String
...@@ -393,6 +400,7 @@ input PageUpdateInput { ...@@ -393,6 +400,7 @@ input PageUpdateInput {
description: String description: String
icon: String icon: String
isBrowsable: Boolean isBrowsable: Boolean
isSearchable: Boolean
locale: String locale: String
password: String password: String
path: String path: String
......
# ===============================================
# SEARCH
# ===============================================
extend type Mutation {
rebuildSearchIndex: DefaultResponse
}
# -----------------------------------------------
# TYPES
# -----------------------------------------------
...@@ -29,6 +29,8 @@ extend type Mutation { ...@@ -29,6 +29,8 @@ extend type Mutation {
key: String! key: String!
): DefaultResponse ): DefaultResponse
rebuildSearchIndex: DefaultResponse
retryJob( retryJob(
id: UUID! id: UUID!
): DefaultResponse ): DefaultResponse
......
...@@ -525,7 +525,7 @@ ...@@ -525,7 +525,7 @@
"admin.scheduler.waitUntil": "Start", "admin.scheduler.waitUntil": "Start",
"admin.search.configSaveSuccess": "Search engine configuration saved successfully.", "admin.search.configSaveSuccess": "Search engine configuration saved successfully.",
"admin.search.dictOverrides": "PostgreSQL Dictionary Mapping Overrides", "admin.search.dictOverrides": "PostgreSQL Dictionary Mapping Overrides",
"admin.search.dictOverridesHint": "JSON object of 2 letters locale codes and their PostgreSQL dictionary association. e.g. { \"en\": \"english\" }", "admin.search.dictOverridesHint": "JSON object of 2 letters locale codes and their PostgreSQL dictionary association. e.g. {0}",
"admin.search.engineConfig": "Engine Configuration", "admin.search.engineConfig": "Engine Configuration",
"admin.search.engineNoConfig": "This engine has no configuration options you can modify.", "admin.search.engineNoConfig": "This engine has no configuration options you can modify.",
"admin.search.highlighting": "Enable Term Highlighting", "admin.search.highlighting": "Enable Term Highlighting",
...@@ -533,10 +533,12 @@ ...@@ -533,10 +533,12 @@
"admin.search.indexRebuildSuccess": "Index rebuilt successfully.", "admin.search.indexRebuildSuccess": "Index rebuilt successfully.",
"admin.search.listRefreshSuccess": "List of search engines has been refreshed.", "admin.search.listRefreshSuccess": "List of search engines has been refreshed.",
"admin.search.rebuildIndex": "Rebuild Index", "admin.search.rebuildIndex": "Rebuild Index",
"admin.search.rebuildInitSuccess": "A search index rebuild has been initiated and will start shortly.",
"admin.search.saveSuccess": "Search engine configuration saved successfully", "admin.search.saveSuccess": "Search engine configuration saved successfully",
"admin.search.searchEngine": "Search Engine", "admin.search.searchEngine": "Search Engine",
"admin.search.subtitle": "Configure the search capabilities of your wiki", "admin.search.subtitle": "Configure the search capabilities of your wiki",
"admin.search.title": "Search Engine", "admin.search.title": "Search Engine",
"admin.searchRebuildIndex": "Rebuild Index",
"admin.security.cors": "CORS (Cross-Origin Resource Sharing)", "admin.security.cors": "CORS (Cross-Origin Resource Sharing)",
"admin.security.corsHostnames": "Hostnames Whitelist", "admin.security.corsHostnames": "Hostnames Whitelist",
"admin.security.corsHostnamesHint": "Enter one hostname per line", "admin.security.corsHostnamesHint": "Enter one hostname per line",
...@@ -1535,6 +1537,7 @@ ...@@ -1535,6 +1537,7 @@
"editor.props.draftHint": "Visible to users with write access only.", "editor.props.draftHint": "Visible to users with write access only.",
"editor.props.icon": "Icon", "editor.props.icon": "Icon",
"editor.props.info": "Info", "editor.props.info": "Info",
"editor.props.isSearchable": "Include in Search Results",
"editor.props.jsLoad": "Javascript - On Load", "editor.props.jsLoad": "Javascript - On Load",
"editor.props.jsLoadHint": "Execute javascript once the page is loaded", "editor.props.jsLoadHint": "Execute javascript once the page is loaded",
"editor.props.jsUnload": "Javascript - On Unload", "editor.props.jsUnload": "Javascript - On Unload",
......
...@@ -344,6 +344,7 @@ export class Page extends Model { ...@@ -344,6 +344,7 @@ export class Page extends Model {
hash: generateHash({ path: opts.path, locale: opts.locale }), hash: generateHash({ path: opts.path, locale: opts.locale }),
icon: opts.icon, icon: opts.icon,
isBrowsable: opts.isBrowsable ?? true, isBrowsable: opts.isBrowsable ?? true,
isSearchable: opts.isSearchable ?? true,
localeCode: opts.locale, localeCode: opts.locale,
ownerId: opts.user.id, ownerId: opts.user.id,
path: opts.path, path: opts.path,
...@@ -388,7 +389,7 @@ export class Page extends Model { ...@@ -388,7 +389,7 @@ export class Page extends Model {
}) })
// -> Update search vector // -> Update search vector
WIKI.db.pages.updatePageSearchVector(page.id) WIKI.db.pages.updatePageSearchVector({ id: page.id })
// // -> Add to Storage // // -> Add to Storage
// if (!opts.skipStorage) { // if (!opts.skipStorage) {
...@@ -507,14 +508,17 @@ export class Page extends Model { ...@@ -507,14 +508,17 @@ export class Page extends Model {
historyData.affectedFields.push('publishEndDate') historyData.affectedFields.push('publishEndDate')
} }
// -> Page Config // -> Browsable / Searchable Flags
if ('isBrowsable' in opts.patch) { if ('isBrowsable' in opts.patch) {
patch.config = { patch.isBrowsable = opts.patch.isBrowsable
...patch.config ?? ogPage.config ?? {},
isBrowsable: opts.patch.isBrowsable
}
historyData.affectedFields.push('isBrowsable') historyData.affectedFields.push('isBrowsable')
} }
if ('isSearchable' in opts.patch) {
patch.isSearchable = opts.patch.isSearchable
historyData.affectedFields.push('isSearchable')
}
// -> Page Config
if ('allowComments' in opts.patch) { if ('allowComments' in opts.patch) {
patch.config = { patch.config = {
...patch.config ?? ogPage.config ?? {}, ...patch.config ?? ogPage.config ?? {},
...@@ -644,7 +648,7 @@ export class Page extends Model { ...@@ -644,7 +648,7 @@ export class Page extends Model {
// -> Save Tags // -> Save Tags
if (opts.patch.tags) { if (opts.patch.tags) {
await WIKI.db.tags.associateTags({ tags: opts.patch.tags, page }) // await WIKI.db.tags.associateTags({ tags: opts.patch.tags, page })
} }
// -> Render page to HTML // -> Render page to HTML
...@@ -672,7 +676,7 @@ export class Page extends Model { ...@@ -672,7 +676,7 @@ export class Page extends Model {
// -> Update search vector // -> Update search vector
if (shouldUpdateSearch) { if (shouldUpdateSearch) {
WIKI.db.pages.updatePageSearchVector(page.id) WIKI.db.pages.updatePageSearchVector({ id: page.id })
} }
// -> Update on Storage // -> Update on Storage
...@@ -710,13 +714,21 @@ export class Page extends Model { ...@@ -710,13 +714,21 @@ export class Page extends Model {
/** /**
* Update a page text search vector value * Update a page text search vector value
* *
* @param {String} id Page UUID * @param {Object} opts - Options
* @param {string} [opts.id] - Page ID to update (fetch from DB)
* @param {Object} [opts.page] - Page object to update (use directly)
*/ */
static async updatePageSearchVector (id) { static async updatePageSearchVector ({ id, page }) {
const page = await WIKI.db.pages.query().findById(id).select('localeCode', 'render') if (!page) {
const safeContent = WIKI.db.pages.cleanHTML(page.render) if (!id) {
throw new Error('Must provide either the page ID or the page object.')
}
page = await WIKI.db.pages.query().findById(id).select('id', 'localeCode', 'render', 'password')
}
// -> Exclude password-protected content from being indexed
const safeContent = page.password ? '' : WIKI.db.pages.cleanHTML(page.render)
const dictName = getDictNameFromLocale(page.localeCode) const dictName = getDictNameFromLocale(page.localeCode)
return WIKI.db.knex('pages').where('id', id).update({ return WIKI.db.knex('pages').where('id', page.id).update({
searchContent: safeContent, searchContent: safeContent,
ts: WIKI.db.knex.raw(` ts: WIKI.db.knex.raw(`
setweight(to_tsvector('${dictName}', coalesce(title,'')), 'A') || setweight(to_tsvector('${dictName}', coalesce(title,'')), 'A') ||
...@@ -726,6 +738,19 @@ export class Page extends Model { ...@@ -726,6 +738,19 @@ export class Page extends Model {
} }
/** /**
* Refresh Autocomplete Index
*/
static async refreshAutocompleteIndex () {
await WIKI.db.knex('autocomplete').truncate()
await WIKI.db.knex.raw(`
INSERT INTO "autocomplete" (word)
SELECT word FROM ts_stat(
'SELECT to_tsvector(''simple'', "title") || to_tsvector(''simple'', "description") || to_tsvector(''simple'', "searchContent") FROM "pages" WHERE isSearchableComputed IS TRUE'
)
`)
}
/**
* Convert an Existing Page * Convert an Existing Page
* *
* @param {Object} opts Page Properties * @param {Object} opts Page Properties
......
export async function task (payload) {
WIKI.logger.info('Refreshing autocomplete word index...')
try {
await WIKI.db.pages.refreshAutocompleteIndex()
WIKI.logger.info('Refreshed autocomplete word index: [ COMPLETED ]')
} catch (err) {
WIKI.logger.error('Refreshing autocomplete word index: [ FAILED ]')
WIKI.logger.error(err.message)
throw err
}
}
...@@ -22,5 +22,6 @@ export async function task ({ payload }) { ...@@ -22,5 +22,6 @@ export async function task ({ payload }) {
} catch (err) { } catch (err) {
WIKI.logger.error('Purging orphaned upload files: [ FAILED ]') WIKI.logger.error('Purging orphaned upload files: [ FAILED ]')
WIKI.logger.error(err.message) WIKI.logger.error(err.message)
throw err
} }
} }
import { pipeline } from 'node:stream/promises'
import { Transform } from 'node:stream'
export async function task ({ payload }) {
WIKI.logger.info('Rebuilding search index...')
try {
await WIKI.ensureDb()
let idx = 0
await pipeline(
WIKI.db.knex.select('id', 'title', 'description', 'localeCode', 'render', 'password').from('pages').stream(),
new Transform({
objectMode: true,
transform: async (page, enc, cb) => {
idx++
await WIKI.db.pages.updatePageSearchVector({ page })
if (idx % 50 === 0) {
WIKI.logger.info(`Rebuilding search index... (${idx} processed)`)
}
cb()
}
})
)
WIKI.logger.info('Refreshing autocomplete index...')
await WIKI.db.pages.refreshAutocompleteIndex()
WIKI.logger.info('Rebuilt search index: [ COMPLETED ]')
} catch (err) {
WIKI.logger.error('Rebuilding search index: [ FAILED ]')
WIKI.logger.error(err.message)
throw err
}
}
...@@ -267,6 +267,15 @@ q-card.page-properties-dialog ...@@ -267,6 +267,15 @@ q-card.page-properties-dialog
) )
div div
q-toggle( q-toggle(
v-model='pageStore.isSearchable'
dense
:label='$t(`editor.props.isSearchable`)'
color='primary'
checked-icon='las la-check'
unchecked-icon='las la-times'
)
div
q-toggle(
v-model='state.requirePassword' v-model='state.requirePassword'
@update:model-value='toggleRequirePassword' @update:model-value='toggleRequirePassword'
dense dense
......
...@@ -6,7 +6,16 @@ q-page.admin-flags ...@@ -6,7 +6,16 @@ q-page.admin-flags
.col.q-pl-md .col.q-pl-md
.text-h5.text-primary.animated.fadeInLeft {{ t('admin.search.title') }} .text-h5.text-primary.animated.fadeInLeft {{ t('admin.search.title') }}
.text-subtitle1.text-grey.animated.fadeInLeft.wait-p2s {{ t('admin.search.subtitle') }} .text-subtitle1.text-grey.animated.fadeInLeft.wait-p2s {{ t('admin.search.subtitle') }}
.col-auto .col-auto.flex
q-btn.q-mr-sm.acrylic-btn(
flat
icon='mdi-database-refresh'
:label='t(`admin.searchRebuildIndex`)'
color='purple'
@click='rebuild'
:loading='state.rebuildLoading'
)
q-separator.q-mr-sm(vertical)
q-btn.q-mr-sm.acrylic-btn( q-btn.q-mr-sm.acrylic-btn(
icon='las la-question-circle' icon='las la-question-circle'
flat flat
...@@ -62,7 +71,9 @@ q-page.admin-flags ...@@ -62,7 +71,9 @@ q-page.admin-flags
language='json' language='json'
:min-height='250' :min-height='250'
) )
q-item-label(caption) {{ t('admin.search.dictOverridesHint') }} q-item-label(caption)
i18n-t(keypath='admin.search.dictOverridesHint' tag='span')
span { "en": "english" }
.col-12.col-lg-5.gt-md .col-12.col-lg-5.gt-md
.q-pa-md.text-center .q-pa-md.text-center
...@@ -104,6 +115,7 @@ useMeta({ ...@@ -104,6 +115,7 @@ useMeta({
const state = reactive({ const state = reactive({
loading: 0, loading: 0,
rebuildLoading: false,
config: { config: {
termHighlighting: false, termHighlighting: false,
dictOverrides: '' dictOverrides: ''
...@@ -181,6 +193,41 @@ async function save () { ...@@ -181,6 +193,41 @@ async function save () {
state.loading-- state.loading--
} }
async function rebuild () {
state.rebuildLoading = true
try {
const respRaw = await APOLLO_CLIENT.mutate({
mutation: gql`
mutation rebuildSearchIndex {
rebuildSearchIndex {
operation {
succeeded
slug
message
}
}
}
`
})
const resp = respRaw?.data?.rebuildSearchIndex?.operation || {}
if (resp.succeeded) {
$q.notify({
type: 'positive',
message: t('admin.search.rebuildInitSuccess')
})
} else {
throw new Error(resp.message)
}
} catch (err) {
$q.notify({
type: 'negative',
message: 'Failed to initiate a search index rebuild',
caption: err.message
})
}
state.rebuildLoading = false
}
// MOUNTED // MOUNTED
onMounted(async () => { onMounted(async () => {
......
...@@ -21,7 +21,7 @@ q-layout(view='hHh Lpr lff') ...@@ -21,7 +21,7 @@ q-layout(view='hHh Lpr lff')
side side
) )
q-icon( q-icon(
:name='state.params.orderByDirection === `desc` ? `mdi-chevron-double-down` : `mdi-chevron-double-up`' :name='state.params.orderByDirection === `desc` ? `mdi-transfer-down` : `mdi-transfer-up`'
size='sm' size='sm'
color='primary' color='primary'
) )
......
...@@ -19,6 +19,7 @@ const pagePropsFragment = gql` ...@@ -19,6 +19,7 @@ const pagePropsFragment = gql`
icon icon
id id
isBrowsable isBrowsable
isSearchable
locale locale
password password
path path
...@@ -122,6 +123,7 @@ const gqlMutations = { ...@@ -122,6 +123,7 @@ const gqlMutations = {
$editor: String! $editor: String!
$icon: String $icon: String
$isBrowsable: Boolean $isBrowsable: Boolean
$isSearchable: Boolean
$locale: String! $locale: String!
$path: String! $path: String!
$publishState: PagePublishState! $publishState: PagePublishState!
...@@ -149,6 +151,7 @@ const gqlMutations = { ...@@ -149,6 +151,7 @@ const gqlMutations = {
editor: $editor editor: $editor
icon: $icon icon: $icon
isBrowsable: $isBrowsable isBrowsable: $isBrowsable
isSearchable: $isSearchable
locale: $locale locale: $locale
path: $path path: $path
publishState: $publishState publishState: $publishState
...@@ -195,6 +198,7 @@ export const usePageStore = defineStore('page', { ...@@ -195,6 +198,7 @@ export const usePageStore = defineStore('page', {
icon: 'las la-file-alt', icon: 'las la-file-alt',
id: '', id: '',
isBrowsable: true, isBrowsable: true,
isSearchable: true,
locale: 'en', locale: 'en',
password: '', password: '',
path: '', path: '',
...@@ -367,6 +371,8 @@ export const usePageStore = defineStore('page', { ...@@ -367,6 +371,8 @@ export const usePageStore = defineStore('page', {
tags: [], tags: [],
content: content ?? '', content: content ?? '',
render: '', render: '',
isBrowsable: true,
isSearchable: true,
mode: 'edit' mode: 'edit'
}) })
}, },
...@@ -420,6 +426,7 @@ export const usePageStore = defineStore('page', { ...@@ -420,6 +426,7 @@ export const usePageStore = defineStore('page', {
'description', 'description',
'icon', 'icon',
'isBrowsable', 'isBrowsable',
'isSearchable',
'locale', 'locale',
'password', 'password',
'path', 'path',
...@@ -491,6 +498,7 @@ export const usePageStore = defineStore('page', { ...@@ -491,6 +498,7 @@ export const usePageStore = defineStore('page', {
'description', 'description',
'icon', 'icon',
'isBrowsable', 'isBrowsable',
'isSearchable',
'locale', 'locale',
'password', 'password',
'path', 'path',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment