feat: Re-support for Elasticsearch

.config/example.yml

@@ -186,6 +186,20 @@ redis:
 #  index: ''
 #  scope: local
 
+#   ┌───────────────────────────┐
+#───┘ ElasticSearch configuration └─────────────────────────────
+
+# If MeiliSearch is also enabled, the MeiliSearch feature is used 
+# and ElasticSearch is disabled.
+
+#elasticsearch:
+#  host: localhost
+#  port: 9200
+#  user: ''
+#  pass: ''
+#  ssl: true
+#  index: ''
+
 #   ┌───────────────┐
 #───┘ ID generation └───────────────────────────────────────────
 

packages/backend/package.json

@@ -71,6 +71,7 @@
 		"@bull-board/fastify": "5.17.0",
 		"@bull-board/ui": "5.17.0",
 		"@discordapp/twemoji": "15.0.3",
+    "@elastic/elasticsearch": "8.13.1",
 		"@fastify/accepts": "4.3.0",
 		"@fastify/cookie": "9.3.1",
 		"@fastify/cors": "9.0.1",

packages/backend/src/GlobalModule.ts

@@ -6,6 +6,7 @@
 import { Global, Inject, Module } from '@nestjs/common';
 import * as Redis from 'ioredis';
 import { DataSource } from 'typeorm';
+import { Client as ElasticSearch } from '@elastic/elasticsearch';
 import { MeiliSearch } from 'meilisearch';
 import { DI } from './di-symbols.js';
 import { Config, loadConfig } from './config.js';
@@ -43,6 +44,25 @@ const $meilisearch: Provider = {
 	inject: [DI.config],
 };
 
+const $elasticsearch: Provider = {
+	provide: DI.elasticsearch,
+	useFactory: (config: Config) => {
+		if (config.elasticsearch) {
+			return new ElasticSearch({
+				nodes: `${config.elasticsearch.ssl ? 'https' : 'http'}://${config.elasticsearch.host}:${config.elasticsearch.port}`,
+				auth: {
+					username: config.elasticsearch.user,
+					password: config.elasticsearch.pass,
+				},
+				//headers: {'Content-Type': 'application/json'},
+			});
+		} else {
+			return null;
+		}
+	},
+	inject: [DI.config],
+};
+
 const $redis: Provider = {
 	provide: DI.redis,
 	useFactory: (config: Config) => {
@@ -81,8 +101,8 @@ const $redisForTimelines: Provider = {
 @Global()
 @Module({
 	imports: [RepositoryModule],
-	providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines],
-	exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule],
+	providers: [$config, $db, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines],
+	exports: [$config, $db, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule],
 })
 export class GlobalModule implements OnApplicationShutdown {
 	constructor(

packages/backend/src/config.ts

@@ -57,6 +57,14 @@ type Source = {
 		index: string;
 		scope?: 'local' | 'global' | string[];
 	};
+	elasticsearch?: {
+		host: string;
+		port: string;
+		user: string;
+		pass: string;
+		ssl?: boolean;
+		index: string;
+	};
 	sentryForBackend?: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; };
 	sentryForFrontend?: { options: Partial<Sentry.NodeOptions> };
 
@@ -129,6 +137,14 @@ export type Config = {
 		index: string;
 		scope?: 'local' | 'global' | string[];
 	} | undefined;
+	elasticsearch: {
+		host: string;
+		port: string;
+		user: string;
+		pass: string;
+		ssl?: boolean;
+		index: string;
+	} | undefined;
 	proxy: string | undefined;
 	proxySmtp: string | undefined;
 	proxyBypassHosts: string[] | undefined;
@@ -235,6 +251,7 @@ export function loadConfig(): Config {
 		dbReplications: config.dbReplications,
 		dbSlaves: config.dbSlaves,
 		meilisearch: config.meilisearch,
+		elasticsearch: config.elasticsearch,
 		redis,
 		redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis,
 		redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis,

packages/backend/src/core/SearchService.ts

@@ -16,6 +16,7 @@ import { isUserRelated } from '@/misc/is-user-related.js';
 import { CacheService } from '@/core/CacheService.js';
 import { QueryService } from '@/core/QueryService.js';
 import { IdService } from '@/core/IdService.js';
+import { Client as ElasticSearch } from '@elastic/elasticsearch';
 import type { Index, MeiliSearch } from 'meilisearch';
 
 type K = string;
@@ -65,6 +66,7 @@ function compileQuery(q: Q): string {
 export class SearchService {
 	private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local';
 	private meilisearchNoteIndex: Index | null = null;
+	private elasticsearchNoteIndex: string | null = null;
 
 	constructor(
 		@Inject(DI.config)
@@ -73,6 +75,9 @@ export class SearchService {
 		@Inject(DI.meilisearch)
 		private meilisearch: MeiliSearch | null,
 
+		@Inject(DI.elasticsearch)
+		private elasticsearch: ElasticSearch | null,
+
 		@Inject(DI.notesRepository)
 		private notesRepository: NotesRepository,
 
@@ -106,6 +111,44 @@ export class SearchService {
 			});
 		}
 
+		if (!meilisearch && this.elasticsearch) {
+			const indexName = `${config.elasticsearch!.index}---notes`;
+			this.elasticsearchNoteIndex = indexName;
+
+			this.elasticsearch.indices.exists({
+				index: indexName,
+			}).then((indexExists) => {
+				if (!indexExists) {
+					this.elasticsearch?.indices.create({
+						index: indexName + `-${new Date().toISOString().slice(0, 7).replace(/-/g, '')}`,
+						body: {
+							mappings: {
+								properties: {
+									text: { type: 'text' },
+									cw: { type: 'text' },
+									createdAt: { type: 'long' },
+									userId: { type: 'keyword' },
+									userHost: { type: 'keyword' },
+									channelId: { type: 'keyword' },
+									tags: { type: 'keyword' },
+								},
+							},
+							settings: {
+								//TODO: Make settings for optimization.
+							},
+						},
+					}).catch((error) => {
+						console.error(error);
+					});
+				}
+			}).catch((error) => {
+				console.error(error);
+			});
+		} else {
+			console.error('Elasticsearch is not available');
+			this.elasticsearchNoteIndex = null;
+		}
+
 		if (config.meilisearch?.scope) {
 			this.meilisearchIndexScope = config.meilisearch.scope;
 		}
@@ -145,6 +188,24 @@ export class SearchService {
 				primaryKey: 'id',
 			});
 		}
+
+		if (!this.meilisearch && this.elasticsearch) {
+			const body = {
+				createdAt: this.idService.parse(note.id).date.getTime(),
+				userId: note.userId,
+				userHost: note.userHost,
+				channelId: note.channelId,
+				cw: note.cw,
+				text: note.text,
+				tags: note.tags,
+			};
+
+			await this.elasticsearch.index({
+				index: this.elasticsearchNoteIndex + `-${new Date().toISOString().slice(0, 7).replace(/-/g, '')}` as string,
+				id: note.id,
+				body: body,
+			});
+		}
 	}
 
 	@bindThis
@@ -154,6 +215,13 @@ export class SearchService {
 		if (this.meilisearch) {
 			this.meilisearchNoteIndex!.deleteDocument(note.id);
 		}
+
+		if (!this.meilisearch && this.elasticsearch) {
+			(this.elasticsearch.delete)({
+				index: this.elasticsearchNoteIndex + `-${new Date().toISOString().slice(0, 7).replace(/-/g, '')}` as string,
+				id: note.id,
+			});
+		}
 	}
 
 	@bindThis
@@ -205,6 +273,65 @@ export class SearchService {
 				return true;
 			});
 			return notes.sort((a, b) => a.id > b.id ? -1 : 1);
+		} else if (this.elasticsearch) {
+			const esFilter: any = {
+				bool: {
+					must: [],
+				},
+			};
+			if (pagination.untilId) esFilter.bool.must.push({ range: { createdAt: { lt: this.idService.parse(pagination.untilId).date.getTime() } } });
+			if (pagination.sinceId) esFilter.bool.must.push({ range: { createdAt: { gt: this.idService.parse(pagination.sinceId).date.getTime() } } });
+			if (opts.userId) esFilter.bool.must.push({ term: { userId: opts.userId } });
+			if (opts.channelId) esFilter.bool.must.push({ term: { channelId: opts.channelId } });
+			if (opts.host) {
+				if (opts.host === '.') {
+					esFilter.bool.must.push({ bool: { must_not: [{ exists: { field: 'userHost' } }] } });
+				} else {
+					esFilter.bool.must.push({ term: { userHost: opts.host } });
+				}
+			}
+			const res = await (this.elasticsearch.search)({
+				index: this.elasticsearchNoteIndex + `*` as string,
+				body: {
+					query: {
+						bool: {
+							must: [
+									{
+									bool: {
+										should: [
+											{ wildcard: { "text": { value: `*${q}*` }, } },
+											{ simple_query_string: { fields: ["text"], "query": q, default_operator: 'and', } },
+										],
+										minimum_should_match: 1,
+									},
+								},
+								esFilter,
+							]
+						},
+					},
+				},
+				sort: [{ createdAt: { order: "desc" } }],
+				_source: ['id', 'createdAt'],
+				size: pagination.limit,
+
+			});
+			const noteIds = res.hits.hits.map((hit: any) => hit._id);
+			if (noteIds.length === 0) return [];
+			const [
+				userIdsWhoMeMuting,
+				userIdsWhoBlockingMe,
+			] = me ? await Promise.all([
+				this.cacheService.userMutingsCache.fetch(me.id),
+				this.cacheService.userBlockedCache.fetch(me.id),
+			]) : [new Set<string>(), new Set<string>()];
+			const notes = (await this.notesRepository.findBy({
+				id: In(noteIds),
+			})).filter(note => {
+				if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false;
+				if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
+				return true;
+			});
+			return notes.sort((a, b) => a.id > b.id ? -1 : 1);
 		} else {
 			const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId);
 

packages/backend/src/di-symbols.ts

@@ -7,6 +7,7 @@ export const DI = {
 	config: Symbol('config'),
 	db: Symbol('db'),
 	meilisearch: Symbol('meilisearch'),
+	elasticsearch: Symbol('elasticsearch'),
 	redis: Symbol('redis'),
 	redisForPub: Symbol('redisForPub'),
 	redisForSub: Symbol('redisForSub'),