Add Meilisearch endpoint (#611)

* add meilisearch to project

* set up endpoint for search and sync

* fix issues with sync and refactor

* added simple documentation

* removed missing refactor

* handle createIndex at endpoint instead

* big refactor, use federated search instead

* add language support and searchable attributes

* move sync endpoint to sync function

* improve type definitions

* small tweak to sync and update comments

* refactor searchable attribute getter and docs

---------

Co-authored-by: Daniel Adu-Gyan <26229521+danieladugyan@users.noreply.github.com>

.env

@@ -65,3 +65,8 @@ PUBLIC_VERCEL_ENV= # set when the app is deployed on Vercel
 MOCK_IS_APP=false # if true, will pretend like all traffic is coming from the app to test how custom app design looks
 REQUEST_LOGGING=false # will log all requests in production
 PRISMA_LOG_LEVEL="silent" # "silent" | "writes" | "all", logs database queries
+
+
+# MEILISEARCH
+MEILISEARCH_HOST=http://localhost:7700
+MEILISEARCH_MASTER_KEY=masterKey

.gitignore

@@ -16,4 +16,5 @@ docs/.vitepress/cache
 /test-results/
 /playwright-report/
 /blob-report/
-/playwright/.cache/
+/playwright/.cache/
+data.ms

dev/setup_db.sh

@@ -1,12 +1,5 @@
 #!/bin/bash
-docker run \
-       --name dsek-db \
-       --publish 5432:5432 \
-       --env POSTGRES_USER=postgres \
-       --env POSTGRES_PASSWORD=postgres \
-       --env POSTGRES_DB=dsek \
-       --detach \
-       postgres:14-alpine
+docker compose up -d
 
 echo "\nWaiting for database to start..."
 until docker exec dsek-db pg_isready

docker-compose.yml

@@ -13,6 +13,19 @@ services:
       - "5432:5432"
     volumes:
       - db:/var/lib/postgresql/data
+
+  meilisearch:
+    container_name: dsek-meilisearch
+    image: getmeili/meilisearch:latest
+    restart: always
+    environment:
+      - MEILI_MASTER_KEY=masterKey
+      - MEILI_NO_ANALYTICS=true
+    ports:
+      - "7700:7700"
+    volumes:
+      - ./data.ms:/data.ms
+
 volumes:
   db:
     driver: local

docs/reference/external-systems.md

@@ -21,3 +21,7 @@ Shlink is used to create short links. We provide an interface for members to cre
 ### MinIO
 
 MinIO is used to store files like profile pictures and meeting documents.
+
+### Meilisearch
+
+Meilisearch is used to provide a good search experience. It can index data from our database and provide rich search capabilities.

package.json

@@ -124,6 +124,7 @@
     "isomorphic-dompurify": "^2.16.0",
     "markdown-to-txt": "^2.0.1",
     "marked": "^11.2.0",
+    "meilisearch": "^0.45.0",
     "minio": "^7.1.3",
     "node-schedule": "^2.1.1",
     "nodemailer": "^6.9.14",

src/app.d.ts

@@ -1,5 +1,6 @@
 import type { ToastNotification } from "$lib/stores/toast";
 import type { Theme } from "$lib/utils/themes";
+import type { AvailableLanguageTag } from "$paraglide/runtime";
 import type { Member, PrismaClient } from "@prisma/client";
 import type { AuthUser } from "@zenstackhq/runtime";
 
@@ -32,6 +33,7 @@ declare global {
       isApp: boolean;
       appInfo?: AppInfo;
       theme: Theme;
+      language: AvailableLanguageTag;
     }
     interface PageData {
       user?: AuthUser;

src/hooks.server.ts

@@ -21,6 +21,7 @@ import loggingExtension from "./database/prisma/loggingExtension";
 import translatedExtension from "./database/prisma/translationExtension";
 import { getAccessPolicies } from "./hooks.server.helpers";
 import { getDerivedRoles } from "$lib/utils/authorization";
+import meilisearchSync from "$lib/search/sync";
 
 const { handle: authHandle } = SvelteKitAuth({
   secret: env.AUTH_SECRET,
@@ -92,6 +93,7 @@ const databaseHandle: Handle = async ({ event, resolve }) => {
   const lang = isAvailableLanguageTag(event.locals.paraglide?.lang)
     ? event.locals.paraglide?.lang
     : sourceLanguageTag;
+  event.locals.language = lang;
   const session = await event.locals.getSession();
   const prisma = prismaClient
     .$extends(translatedExtension(lang))
@@ -210,6 +212,7 @@ const themeHandle: Handle = async ({ event, resolve }) => {
 
 // run a keycloak sync every day at midnight
 schedule.scheduleJob("0 0 * * *", () => keycloak.sync(authorizedPrismaClient));
+schedule.scheduleJob("0 0 * * *", meilisearchSync);
 
 export const handle = sequence(
   authHandle,

src/lib/search/meilisearch.ts

@@ -0,0 +1,12 @@
+import { Meilisearch } from "meilisearch";
+import { env } from "$env/dynamic/private";
+
+export const meilisearch = new Meilisearch({
+  host: env.MEILISEARCH_HOST,
+  apiKey: env.MEILISEARCH_MASTER_KEY,
+  requestConfig: {
+    headers: {
+      Authorization: `Bearer ${env.MEILISEARCH_MASTER_KEY}`,
+    },
+  },
+});

src/lib/search/searchHelpers.ts

@@ -0,0 +1,103 @@
+import type { AvailableLanguageTag } from "$paraglide/runtime";
+import {
+  availableSearchIndexes,
+  type SearchableArticleAttributes,
+  type SearchableEventAttributes,
+  type SearchableIndex,
+  type SearchableMemberAttributes,
+  type SearchablePositionAttributes,
+  type SearchableSongAttributes,
+} from "./searchTypes";
+
+export type SearchIndex =
+  (typeof availableSearchIndexes)[keyof typeof availableSearchIndexes];
+
+/**
+ * Get the federated weight for a given index
+ * This is used to give a higher weight to certain indexes
+ * when searching. For example, we want members to be more
+ * important than events. This is done by giving members
+ * a higher weight.
+ */
+export function getFederatedWeight(index: SearchableIndex): number {
+  switch (index) {
+    case "members":
+      return 5;
+    case "events":
+      return 1;
+    case "articles":
+      return 1;
+    case "positions":
+      return 5;
+    case "songs":
+      return 1;
+    default:
+      return 1;
+  }
+}
+
+/**
+ * Get the searchable attributes for a given index and language
+ * If for whatever reason the language is not supported, or our
+ * systems fail to provide a language, swedish will be used as default.
+ */
+export function getSearchableAttributes(
+  index: SearchableIndex,
+  language: AvailableLanguageTag,
+) {
+  switch (index) {
+    case "members": {
+      // no language specific fields
+      const res: Array<keyof SearchableMemberAttributes> = [
+        "firstName",
+        "lastName",
+        "nickname",
+        "studentId",
+        "fullName",
+      ];
+      return res;
+    }
+    case "events": {
+      // default is swedish
+      let res: Array<keyof SearchableEventAttributes> = [
+        "title",
+        "description",
+      ];
+      if (language === "en") {
+        res = ["titleEn", "descriptionEn"];
+      }
+      return res;
+    }
+    case "articles": {
+      // default is swedish
+      let res: Array<keyof SearchableArticleAttributes> = ["header", "body"];
+      if (language === "en") {
+        res = ["headerEn", "bodyEn"];
+      }
+      return res;
+    }
+    case "positions": {
+      // default is swedish
+      let res: Array<keyof SearchablePositionAttributes> = [
+        "name",
+        "description",
+        "committeeName",
+        "dsekId",
+      ];
+      if (language === "en") {
+        res = ["nameEn", "descriptionEn", "committeeNameEn", "dsekId"];
+      }
+      return res;
+    }
+    case "songs": {
+      // no language specific fields
+      const res: Array<keyof SearchableSongAttributes> = [
+        "title",
+        "lyrics",
+        "melody",
+        "category",
+      ];
+      return res;
+    }
+  }
+}

src/lib/search/searchTypes.ts

@@ -0,0 +1,130 @@
+import type {
+  Article,
+  Member,
+  Event,
+  Song,
+  Position,
+  Committee,
+} from "@prisma/client";
+
+/**
+ * Utility type that creates a new object type based on a union of keys (Keys).
+ *
+ * For each key in Keys:
+ *   - If the key exists in T, its value type is preserved from T.
+ *   - If the key does not exist in T, its value type is set to `string`.
+ *
+ * This is useful for creating a new type that includes specific keys (from Keys),
+ * ensuring compatibility with an existing type (T), while accounting for missing keys.
+ */
+type FilterKeys<T extends Record<string, unknown>, Keys extends string> = {
+  [Key in Keys]: Key extends keyof T ? T[Key] : string;
+};
+
+export const availableSearchIndexes = [
+  "members",
+  "events",
+  "articles",
+  "positions",
+  "songs",
+] as const;
+export type SearchableIndex = (typeof availableSearchIndexes)[number];
+
+export const memberSearchableAttributes = [
+  "id",
+  "firstName",
+  "lastName",
+  "nickname",
+  "studentId",
+  "fullName",
+] as const satisfies Array<keyof Member | "fullName">;
+export type SearchableMemberAttributes = FilterKeys<
+  Member,
+  (typeof memberSearchableAttributes)[number]
+>;
+
+export const eventSearchableAttributes = [
+  "title",
+  "titleEn",
+  "description",
+  "descriptionEn",
+] as const satisfies Array<keyof Event>;
+export type SearchableEventAttributes = Pick<
+  Event,
+  (typeof eventSearchableAttributes)[number]
+>;
+
+export const articleSearchableAttributes = [
+  "header",
+  "headerEn",
+  "body",
+  "bodyEn",
+] as const satisfies Array<keyof Article>;
+export type SearchableArticleAttributes = Pick<
+  Article,
+  (typeof articleSearchableAttributes)[number]
+>;
+
+export const positionSearchableAttributes = [
+  "name",
+  "nameEn",
+  "description",
+  "descriptionEn",
+  "dsekId",
+  "committeeName",
+  "committeeNameEn",
+] as const satisfies Array<
+  keyof Position | "dsekId" | "committeeName" | "committeeNameEn"
+>;
+export type SearchablePositionAttributes = FilterKeys<
+  Position,
+  (typeof positionSearchableAttributes)[number]
+>;
+
+export const songSearchableAttributes = [
+  "title",
+  "lyrics",
+  "melody",
+  "category",
+] as const satisfies Array<keyof Song>;
+export type SearchableSongAttributes = Pick<
+  Song,
+  (typeof songSearchableAttributes)[number]
+>;
+
+export type SongSearchReturnAttributes = SearchableSongAttributes &
+  Pick<Song, "slug">;
+export type ArticleSearchReturnAttributes = SearchableArticleAttributes &
+  Pick<Article, "slug">;
+export type EventSearchReturnAttributes = SearchableEventAttributes &
+  Pick<Event, "slug">;
+export type MemberSearchReturnAttributes = SearchableMemberAttributes & {
+  picturePath: string;
+  classYear: number;
+};
+export type PositionSearchReturnAttributes = SearchablePositionAttributes &
+  Pick<Position, "committeeId"> & {
+    committee: Committee | null;
+  };
+
+export type SearchDataWithType =
+  | {
+      type: "members";
+      data: MemberSearchReturnAttributes;
+    }
+  | {
+      type: "events";
+      data: EventSearchReturnAttributes;
+    }
+  | {
+      type: "articles";
+      data: ArticleSearchReturnAttributes;
+    }
+  | {
+      type: "songs";
+      data: SongSearchReturnAttributes;
+    }
+  | {
+      type: "positions";
+      data: PositionSearchReturnAttributes;
+    };