Skip to content

Commit

Permalink
big refactor, use federated search instead
Browse files Browse the repository at this point in the history
  • Loading branch information
alfredgrip committed Dec 2, 2024
1 parent 41659e2 commit f66e8c3
Show file tree
Hide file tree
Showing 6 changed files with 243 additions and 14 deletions.
File renamed without changes.
21 changes: 21 additions & 0 deletions src/lib/search/searchHelpers.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import { availableSearchIndexes } from "./searchTypes";

export type SearchIndex =
(typeof availableSearchIndexes)[keyof typeof availableSearchIndexes];

export function getFederatedWeight(index: string): number {
switch (index) {
case availableSearchIndexes.members:
return 5;
case availableSearchIndexes.events:
return 1;
case availableSearchIndexes.articles:
return 1;
case availableSearchIndexes.positions:
return 5;
case availableSearchIndexes.songs:
return 1;
default:
return 1;
}
}
77 changes: 77 additions & 0 deletions src/lib/search/searchTypes.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import type {
Article,
Member,
Event,
Song,
Committee,
Position,
} from "@prisma/client";

export const availableSearchIndexes = {
members: "members",
events: "events",
articles: "articles",
positions: "positions",
songs: "songs",
} as const;

export type SearchMember = Pick<
Member,
| "studentId"
| "firstName"
| "lastName"
| "nickname"
| "picturePath"
| "classYear"
> & {
name: string;
id: string;
};
export type SearchSong = Pick<
Song,
"title" | "category" | "lyrics" | "melody" | "slug"
> & {
id: string;
};
export type SearchArticle = Pick<
Article,
"body" | "bodyEn" | "header" | "headerEn" | "slug"
> & {
id: string;
};
export type SearchEvent = Pick<
Event,
"title" | "titleEn" | "description" | "descriptionEn" | "slug"
> & {
id: string;
};
export type SearchPosition = Pick<
Position,
"committeeId" | "description" | "descriptionEn" | "name" | "nameEn"
> & {
committee: Committee | null;
id: string;
dsekId: Position["id"];
};

export type SearchDataWithType =
| {
type: "members";
data: SearchMember;
}
| {
type: "events";
data: SearchEvent;
}
| {
type: "articles";
data: SearchArticle;
}
| {
type: "songs";
data: SearchSong;
}
| {
type: "positions";
data: SearchPosition;
};
32 changes: 28 additions & 4 deletions src/routes/(app)/api/search/+server.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import type { RequestHandler } from "@sveltejs/kit";
import { meilisearch } from "./meilisearch";
import { meilisearch } from "$lib/search/meilisearch";
import { getFederatedWeight } from "$lib/search/searchHelpers";
import type { Hits } from "meilisearch";

/**
* This endpoint is used to search multiple indexes at once.
Expand Down Expand Up @@ -35,12 +37,34 @@ export const GET: RequestHandler = async ({ url }) => {
);
}
}
let limit = Number.parseInt(url.searchParams.get("limit") ?? "20");
if (limit === -1) {
limit = 20;
}

const search = await meilisearch.multiSearch({
queries: indexes.map((index) => ({ indexUid: index, q: query, limit: 5 })),
const response = await meilisearch.multiSearch({
queries: indexes.map((index) => ({
indexUid: index,
q: query,
federationOptions: {
weight: getFederatedWeight(index),
},
showRankingScoreDetails: true,
})),
federation: {},
});

return new Response(JSON.stringify(search), {
const array: Hits = [];
let i = 0;
while (array.length < limit && i < response.hits.length) {
const hit = response.hits[i];
if (hit != null) {
array.push(hit);
}
i++;
}

return new Response(JSON.stringify(array), {
headers: {
"Content-Type": "application/json",
},
Expand Down
85 changes: 82 additions & 3 deletions src/routes/(app)/api/search/sync/+server.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
import { meilisearch } from "../meilisearch";
import { meilisearch } from "$lib/search/meilisearch";
import type {
SearchMember,
SearchArticle,
SearchEvent,
SearchPosition,
SearchSong,
} from "$lib/search/searchTypes";
import type { RequestHandler } from "./$types";
import { v4 as uuid } from "uuid";

/**
* Dumps relevant data from the database to Meilisearch.
* Meilisearch basically has its own database, so we need to
Expand Down Expand Up @@ -41,14 +49,22 @@ export const GET: RequestHandler = async ({ locals, getClientAddress }) => {
* Perhaps generating UUIDs isn't needed in a future version of
* Meilisearch.
*/
const [members, songs, articles, events, positions] = await Promise.all([
const [members, songs, articles, events, positions]: [
SearchMember[],
SearchSong[],
SearchArticle[],
SearchEvent[],
SearchPosition[],
] = await Promise.all([
prisma.member
.findMany({
select: {
studentId: true,
firstName: true,
lastName: true,
nickname: true,
picturePath: true,
classYear: true,
},
})
.then((members) =>
Expand All @@ -65,6 +81,7 @@ export const GET: RequestHandler = async ({ locals, getClientAddress }) => {
category: true,
lyrics: true,
melody: true,
slug: true,
},
where: {
deletedAt: null,
Expand All @@ -83,6 +100,7 @@ export const GET: RequestHandler = async ({ locals, getClientAddress }) => {
bodyEn: true,
header: true,
headerEn: true,
slug: true,
},
where: {
AND: [
Expand Down Expand Up @@ -110,6 +128,7 @@ export const GET: RequestHandler = async ({ locals, getClientAddress }) => {
titleEn: true,
description: true,
descriptionEn: true,
slug: true,
},
where: {
AND: [
Expand All @@ -128,6 +147,8 @@ export const GET: RequestHandler = async ({ locals, getClientAddress }) => {
prisma.position
.findMany({
select: {
id: true,
committeeId: true,
committee: true,
description: true,
descriptionEn: true,
Expand All @@ -138,6 +159,8 @@ export const GET: RequestHandler = async ({ locals, getClientAddress }) => {
.then((positions) =>
positions.map((position) => ({
...position,
dsekId: position.id,
committeeName: position.committee?.name ?? "",
id: uuid(),
})),
),
Expand All @@ -160,7 +183,13 @@ export const GET: RequestHandler = async ({ locals, getClientAddress }) => {
positionsIndex.deleteAllDocuments(),
]);

await Promise.all([
const [
addMembersTask,
addSongsTask,
addArticlesTask,
addEventsTask,
positionsTask,
] = await Promise.all([
membersIndex.addDocuments(members, {
primaryKey: "id",
}),
Expand All @@ -178,6 +207,56 @@ export const GET: RequestHandler = async ({ locals, getClientAddress }) => {
}),
]);

// Wait for all add tasks to finish
await meilisearch.waitForTasks(
[
addMembersTask.taskUid,
addSongsTask.taskUid,
addArticlesTask.taskUid,
addEventsTask.taskUid,
positionsTask.taskUid,
],
{
timeOutMs: 10000,
},
);

await Promise.all([
membersIndex.updateSearchableAttributes([
"firstName",
"lastName",
"name",
"studentId",
"nickname",
]),
songsIndex.updateSearchableAttributes([
"title",
"category",
"lyrics",
"melody",
]),
articlesIndex.updateSearchableAttributes([
"header",
"body",
"headerEn",
"bodyEn",
]),
eventsIndex.updateSearchableAttributes([
"title",
"description",
"titleEn",
"descriptionEn",
]),
positionsIndex.updateSearchableAttributes([
"name",
"nameEn",
"committeeName",
"description",
"descriptionEn",
"dsekId",
]),
]);

return new Response(
JSON.stringify({ members, songs, articles, events, positions }),
{
Expand Down
42 changes: 35 additions & 7 deletions src/routes/(app)/search/+page.server.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,42 @@
import type { SearchDataWithType } from "$lib/search/searchTypes";
import type { Actions } from "./$types";
import type { Hits } from "meilisearch";

export const actions = {
default: async (event) => {
const data = await event.request.formData();
const search = data.get("input");
if (typeof search !== "string") return;
const response = await event.fetch(
"/api/members?" + new URLSearchParams({ search }),
);
const users = await response.json();
return { users };

const query = data.get("input");
if (typeof query !== "string") return;

const includeMembers = data.get("members") === "on";
const includeEvents = data.get("events") === "on";
const includeArticles = data.get("articles") === "on";
const includePositions = data.get("positions") === "on";
const includeSongs = data.get("songs") === "on";
const indexes = [];
if (includeMembers) indexes.push("members");
if (includeEvents) indexes.push("events");
if (includeArticles) indexes.push("articles");
if (includePositions) indexes.push("positions");
if (includeSongs) indexes.push("songs");

const url = new URL("/api/search", event.request.url);
url.searchParams.set("query", query);
url.searchParams.set("indexes", JSON.stringify(indexes));
const response = await event.fetch(url);
if (!response.ok) {
// silently fail
return;
}
const json: Hits = await response.json();
return {
results: json.map((hit) => {
return {
data: hit,
type: hit._federation?.indexUid,
} as SearchDataWithType;
}),
};
},
} satisfies Actions;

0 comments on commit f66e8c3

Please sign in to comment.