Merge pull request #3468 from mikiher/nunicode-intergration

Nunicode integration

.gitignore

@@ -16,6 +16,7 @@
 /ffmpeg*
 /ffprobe*
 /unicode*
+/libnusqlite3*
 
 sw.*
 .DS_STORE

Dockerfile

@@ -11,20 +11,36 @@ FROM node:20-alpine
 ENV NODE_ENV=production
 
 RUN apk update && \
- apk add --no-cache --update \
- curl \
- tzdata \
- ffmpeg \
- make \
- gcompat \
- python3 \
- g++ \
- tini
+ apk add --no-cache --update \
+ curl \
+ tzdata \
+ ffmpeg \
+ make \
+ gcompat \
+ python3 \
+ g++ \
+ tini \
+ unzip
 
 COPY --from=build /client/dist /client/dist
 COPY index.js package* /
 COPY server server
 
+ARG TARGETPLATFORM
+
+ENV NUSQLITE3_DIR="/usr/local/lib/nusqlite3"
+ENV NUSQLITE3_PATH="${NUSQLITE3_DIR}/libnusqlite3.so"
+
+RUN case "$TARGETPLATFORM" in \
+ "linux/amd64") \
+ curl -L -o /tmp/library.zip "https://github.com/mikiher/nunicode-sqlite/releases/download/v1.1/libnusqlite3-linux-x64.zip" ;; \
+ "linux/arm64") \
+ curl -L -o /tmp/library.zip "https://github.com/mikiher/nunicode-sqlite/releases/download/v1.1/libnusqlite3-linux-arm64.zip" ;; \
+ *) echo "Unsupported platform: $TARGETPLATFORM" && exit 1 ;; \
+ esac && \
+ unzip /tmp/library.zip -d $NUSQLITE3_DIR && \
+ rm /tmp/library.zip
+
 RUN npm ci --only=production
 
 RUN apk del make python3 g++

index.js

@@ -9,6 +9,7 @@ if (isDev) {
  if (devEnv.MetadataPath) process.env.METADATA_PATH = devEnv.MetadataPath
  if (devEnv.FFmpegPath) process.env.FFMPEG_PATH = devEnv.FFmpegPath
  if (devEnv.FFProbePath) process.env.FFPROBE_PATH = devEnv.FFProbePath
+ if (devEnv.NunicodePath) process.env.NUSQLITE3_PATH = devEnv.NunicodePath
  if (devEnv.SkipBinariesCheck) process.env.SKIP_BINARIES_CHECK = '1'
  if (devEnv.BackupPath) process.env.BACKUP_PATH = devEnv.BackupPath
  process.env.SOURCE = 'local'

server/Database.js

@@ -28,6 +28,9 @@ class Database {
  this.notificationSettings = null
  /** @type {import('./objects/settings/EmailSettings')} */
  this.emailSettings = null
+
+ this.supportsUnaccent = false
+ this.supportsUnicodeFoldings = false
  }
 
  get models() {
@@ -223,6 +226,12 @@ class Database {
 
  try {
  await this.sequelize.authenticate()
+ if (process.env.NUSQLITE3_PATH) {
+ await this.loadExtension(process.env.NUSQLITE3_PATH)
+ Logger.info(`[Database] Db supports unaccent and unicode foldings`)
+ this.supportsUnaccent = true
+ this.supportsUnicodeFoldings = true
+ }
  Logger.info(`[Database] Db connection was successful`)
  return true
  } catch (error) {
@@ -232,31 +241,28 @@ class Database {
  }
 
  /**
- * TODO: Temporarily disabled
- * @param {string[]} extensions paths to extension binaries
+ * @param {string} extension paths to extension binary
  */
- async loadExtensions(extensions) {
+ async loadExtension(extension) {
  // This is a hack to get the db connection for loading extensions.
  // The proper way would be to use the 'afterConnect' hook, but that hook is never called for sqlite due to a bug in sequelize.
  // See https://github.com/sequelize/sequelize/issues/12487
  // This is not a public API and may break in the future.
  const db = await this.sequelize.dialect.connectionManager.getConnection()
  if (typeof db?.loadExtension !== 'function') throw new Error('Failed to get db connection for loading extensions')
 
- for (const ext of extensions) {
- Logger.info(`[Database] Loading extension ${ext}`)
- await new Promise((resolve, reject) => {
- db.loadExtension(ext, (err) => {
- if (err) {
- Logger.error(`[Database] Failed to load extension ${ext}`, err)
- reject(err)
- return
- }
- Logger.info(`[Database] Successfully loaded extension ${ext}`)
- resolve()
- })
+ Logger.info(`[Database] Loading extension ${extension}`)
+ await new Promise((resolve, reject) => {
+ db.loadExtension(extension, (err) => {
+ if (err) {
+ Logger.error(`[Database] Failed to load extension ${extension}`, err)
+ reject(err)
+ return
+ }
+ Logger.info(`[Database] Successfully loaded extension ${extension}`)
+ resolve()
  })
- }
+ })
  }
 
  /**
@@ -745,37 +751,57 @@ class Database {
  }
  }
 
- /**
- * TODO: Temporarily unused
- * @param {string} value
- * @returns {string}
- */
- normalize(value) {
- return `lower(unaccent(${value}))`
+ async createTextSearchQuery(query) {
+ const textQuery = new this.TextSearchQuery(this.sequelize, this.supportsUnaccent, query)
+ await textQuery.init()
+ return textQuery
  }
 
- /**
- * TODO: Temporarily unused
- * @param {string} query
- * @returns {Promise<string>}
- */
- async getNormalizedQuery(query) {
- const escapedQuery = this.sequelize.escape(query)
- const normalizedQuery = this.normalize(escapedQuery)
- const normalizedQueryResult = await this.sequelize.query(`SELECT ${normalizedQuery} as normalized_query`)
- return normalizedQueryResult[0][0].normalized_query
- }
+ TextSearchQuery = class {
+ constructor(sequelize, supportsUnaccent, query) {
+ this.sequelize = sequelize
+ this.supportsUnaccent = supportsUnaccent
+ this.query = query
+ this.hasAccents = false
+ }
 
- /**
- *
- * @param {string} column
- * @param {string} normalizedQuery
- * @returns {string}
- */
- matchExpression(column, normalizedQuery) {
- const normalizedPattern = this.sequelize.escape(`%${normalizedQuery}%`)
- const normalizedColumn = column
- return `${normalizedColumn} LIKE ${normalizedPattern}`
+ /**
+ * Returns a normalized (accents-removed) expression for the specified value.
+ *
+ * @param {string} value
+ * @returns {string}
+ */
+ normalize(value) {
+ return `unaccent(${value})`
+ }
+
+ /**
+ * Initialize the text query.
+ *
+ */
+ async init() {
+ if (!this.supportsUnaccent) return
+ const escapedQuery = this.sequelize.escape(this.query)
+ const normalizedQueryExpression = this.normalize(escapedQuery)
+ const normalizedQueryResult = await this.sequelize.query(`SELECT ${normalizedQueryExpression} as normalized_query`)
+ const normalizedQuery = normalizedQueryResult[0][0].normalized_query
+ this.hasAccents = escapedQuery !== this.sequelize.escape(normalizedQuery)
+ }
+
+ /**
+ * Get match expression for the specified column.
+ * If the query contains accents, match against the column as-is (case-insensitive exact match).
+ * otherwise match against a normalized column (case-insensitive match with accents removed).
+ *
+ * @param {string} column
+ * @returns {string}
+ */
+ matchExpression(column) {
+ const pattern = this.sequelize.escape(`%${this.query}%`)
+ if (!this.supportsUnaccent) return `${column} LIKE ${pattern}`
+ const normalizedColumn = this.hasAccents ? column : this.normalize(column)
+ return `${normalizedColumn} LIKE ${pattern}`
+ }
  }
 }