refactor(server): db env (#13167)

immich-app · Oct 3, 2024 · 3ac00b0 · 3ac00b0
1 parent e2bf680
commit 3ac00b0
Show file tree

Hide file tree

Showing 11 changed files with 84 additions and 31 deletions.
diff --git a/server/src/database.config.ts b/server/src/database.config.ts
@@ -1,16 +1,17 @@
-import { DatabaseExtension } from 'src/interfaces/database.interface';
+import { ConfigRepository } from 'src/repositories/config.repository';
 import { DataSource } from 'typeorm';
 import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions.js';
 
-const url = process.env.DB_URL;
+const { database } = new ConfigRepository().getEnv();
+const { url, host, port, username, password, name } = database;
 const urlOrParts = url
   ? { url }
   : {
-      host: process.env.DB_HOSTNAME || 'database',
-      port: Number.parseInt(process.env.DB_PORT || '5432'),
-      username: process.env.DB_USERNAME || 'postgres',
-      password: process.env.DB_PASSWORD || 'postgres',
-      database: process.env.DB_DATABASE_NAME || 'immich',
+      host,
+      port,
+      username,
+      password,
+      database: name,
     };
 
 /* eslint unicorn/prefer-module: "off" -- We can fix this when migrating to ESM*/
@@ -32,6 +33,3 @@ export const databaseConfig: PostgresConnectionOptions = {
  * this export is ONLY to be used for TypeORM commands in package.json#scripts
  */
 export const dataSource = new DataSource({ ...databaseConfig, host: 'localhost' });
-
-export const getVectorExtension = () =>
-  process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS;
diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.ts
@@ -26,6 +26,12 @@ export interface EnvData {
   };
 
   database: {
+    url?: string;
+    host: string;
+    port: number;
+    username: string;
+    password: string;
+    name: string;
     skipMigrations: boolean;
     vectorExtension: VectorExtension;
   };

diff --git a/server/src/migrations/1700713871511-UsePgVectors.ts b/server/src/migrations/1700713871511-UsePgVectors.ts
@@ -1,13 +1,15 @@
-import { getVectorExtension } from 'src/database.config';
+import { ConfigRepository } from 'src/repositories/config.repository';
 import { getCLIPModelInfo } from 'src/utils/misc';
 import { MigrationInterface, QueryRunner } from 'typeorm';
 
+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
+
 export class UsePgVectors1700713871511 implements MigrationInterface {
   name = 'UsePgVectors1700713871511';
 
   public async up(queryRunner: QueryRunner): Promise<void> {
     await queryRunner.query(`SET search_path TO "$user", public, vectors`);
-    await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${getVectorExtension()}`);
+    await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${vectorExtension}`);
     const faceDimQuery = await queryRunner.query(`
         SELECT CARDINALITY(embedding::real[]) as dimsize
         FROM asset_faces

diff --git a/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts b/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts
@@ -1,12 +1,14 @@
-import { getVectorExtension } from 'src/database.config';
 import { DatabaseExtension } from 'src/interfaces/database.interface';
+import { ConfigRepository } from 'src/repositories/config.repository';
 import { MigrationInterface, QueryRunner } from 'typeorm';
 
+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
+
 export class AddCLIPEmbeddingIndex1700713994428 implements MigrationInterface {
   name = 'AddCLIPEmbeddingIndex1700713994428';
 
   public async up(queryRunner: QueryRunner): Promise<void> {
-    if (getVectorExtension() === DatabaseExtension.VECTORS) {
+    if (vectorExtension === DatabaseExtension.VECTORS) {
       await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
     }
     await queryRunner.query(`SET search_path TO "$user", public, vectors`);

diff --git a/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts b/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts
@@ -1,12 +1,14 @@
-import { getVectorExtension } from 'src/database.config';
 import { DatabaseExtension } from 'src/interfaces/database.interface';
+import { ConfigRepository } from 'src/repositories/config.repository';
 import { MigrationInterface, QueryRunner } from 'typeorm';
 
+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
+
 export class AddFaceEmbeddingIndex1700714033632 implements MigrationInterface {
   name = 'AddFaceEmbeddingIndex1700714033632';
 
   public async up(queryRunner: QueryRunner): Promise<void> {
-    if (getVectorExtension() === DatabaseExtension.VECTORS) {
+    if (vectorExtension === DatabaseExtension.VECTORS) {
       await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
     }
     await queryRunner.query(`SET search_path TO "$user", public, vectors`);

diff --git a/server/src/migrations/1718486162779-AddFaceSearchRelation.ts b/server/src/migrations/1718486162779-AddFaceSearchRelation.ts
@@ -1,10 +1,12 @@
-import { getVectorExtension } from 'src/database.config';
 import { DatabaseExtension } from 'src/interfaces/database.interface';
+import { ConfigRepository } from 'src/repositories/config.repository';
 import { MigrationInterface, QueryRunner } from 'typeorm';
 
+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
+
 export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
   public async up(queryRunner: QueryRunner): Promise<void> {
-    if (getVectorExtension() === DatabaseExtension.VECTORS) {
+    if (vectorExtension === DatabaseExtension.VECTORS) {
       await queryRunner.query(`SET search_path TO "$user", public, vectors`);
       await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
     }
@@ -13,9 +15,10 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
       const columns = await queryRunner.query(
         `SELECT column_name as name
         FROM information_schema.columns
-        WHERE table_name = '${tableName}'`);
+        WHERE table_name = '${tableName}'`,
+      );
       return columns.some((column: { name: string }) => column.name === 'embedding');
-    }
+    };
 
     const hasAssetEmbeddings = await hasEmbeddings('smart_search');
     if (!hasAssetEmbeddings) {
@@ -31,7 +34,7 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
     await queryRunner.query(`ALTER TABLE face_search ALTER COLUMN embedding SET STORAGE EXTERNAL`);
     await queryRunner.query(`ALTER TABLE smart_search ALTER COLUMN embedding SET STORAGE EXTERNAL`);
 
-    const hasFaceEmbeddings = await hasEmbeddings('asset_faces')
+    const hasFaceEmbeddings = await hasEmbeddings('asset_faces');
     if (hasFaceEmbeddings) {
       await queryRunner.query(`
         INSERT INTO face_search("faceId", embedding)
@@ -56,7 +59,7 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
   }
 
   public async down(queryRunner: QueryRunner): Promise<void> {
-    if (getVectorExtension() === DatabaseExtension.VECTORS) {
+    if (vectorExtension === DatabaseExtension.VECTORS) {
       await queryRunner.query(`SET search_path TO "$user", public, vectors`);
       await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
     }

diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts
@@ -1,7 +1,7 @@
 import { Injectable } from '@nestjs/common';
-import { getVectorExtension } from 'src/database.config';
 import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';
 import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';
+import { DatabaseExtension } from 'src/interfaces/database.interface';
 import { setDifference } from 'src/utils/set';
 
 // TODO replace src/config validation with class-validator, here
@@ -65,8 +65,16 @@ export class ConfigRepository implements IConfigRepository {
       },
 
       database: {
+        url: process.env.DB_URL,
+        host: process.env.DB_HOSTNAME || 'database',
+        port: Number(process.env.DB_PORT) || 5432,
+        username: process.env.DB_USERNAME || 'postgres',
+        password: process.env.DB_PASSWORD || 'postgres',
+        name: process.env.DB_DATABASE_NAME || 'immich',
+
         skipMigrations: process.env.DB_SKIP_MIGRATIONS === 'true',
-        vectorExtension: getVectorExtension(),
+        vectorExtension:
+          process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS,
       },
 
       licensePublicKey: isProd ? productionKeys : stagingKeys,

diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts
@@ -3,7 +3,7 @@ import { InjectDataSource } from '@nestjs/typeorm';
 import AsyncLock from 'async-lock';
 import semver from 'semver';
 import { POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants';
-import { getVectorExtension } from 'src/database.config';
+import { IConfigRepository } from 'src/interfaces/config.interface';
 import {
   DatabaseExtension,
   DatabaseLock,
@@ -22,12 +22,15 @@ import { DataSource, EntityManager, QueryRunner } from 'typeorm';
 @Instrumentation()
 @Injectable()
 export class DatabaseRepository implements IDatabaseRepository {
+  private vectorExtension: VectorExtension;
   readonly asyncLock = new AsyncLock();
 
   constructor(
     @InjectDataSource() private dataSource: DataSource,
     @Inject(ILoggerRepository) private logger: ILoggerRepository,
+    @Inject(IConfigRepository) configRepository: IConfigRepository,
   ) {
+    this.vectorExtension = configRepository.getEnv().database.vectorExtension;
     this.logger.setContext(DatabaseRepository.name);
   }
 
@@ -119,7 +122,7 @@ export class DatabaseRepository implements IDatabaseRepository {
     try {
       await this.dataSource.query(`REINDEX INDEX ${index}`);
     } catch (error) {
-      if (getVectorExtension() !== DatabaseExtension.VECTORS) {
+      if (this.vectorExtension !== DatabaseExtension.VECTORS) {
         throw error;
       }
       this.logger.warn(`Could not reindex index ${index}. Attempting to auto-fix.`);
@@ -141,7 +144,7 @@ export class DatabaseRepository implements IDatabaseRepository {
   }
 
   async shouldReindex(name: VectorIndex): Promise<boolean> {
-    if (getVectorExtension() !== DatabaseExtension.VECTORS) {
+    if (this.vectorExtension !== DatabaseExtension.VECTORS) {
       return false;
     }
 

diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts
@@ -1,7 +1,6 @@
 import { Inject, Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { randomUUID } from 'node:crypto';
-import { getVectorExtension } from 'src/database.config';
 import { DummyValue, GenerateSql } from 'src/decorators';
 import { AssetFaceEntity } from 'src/entities/asset-face.entity';
 import { AssetEntity } from 'src/entities/asset.entity';
@@ -10,7 +9,8 @@ import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
 import { SmartInfoEntity } from 'src/entities/smart-info.entity';
 import { SmartSearchEntity } from 'src/entities/smart-search.entity';
 import { AssetType, PaginationMode } from 'src/enum';
-import { DatabaseExtension } from 'src/interfaces/database.interface';
+import { IConfigRepository } from 'src/interfaces/config.interface';
+import { DatabaseExtension, VectorExtension } from 'src/interfaces/database.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import {
   AssetDuplicateResult,
@@ -31,6 +31,7 @@ import { Repository, SelectQueryBuilder } from 'typeorm';
 @Instrumentation()
 @Injectable()
 export class SearchRepository implements ISearchRepository {
+  private vectorExtension: VectorExtension;
   private faceColumns: string[];
   private assetsByCityQuery: string;
 
@@ -42,7 +43,9 @@ export class SearchRepository implements ISearchRepository {
     @InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository<SmartSearchEntity>,
     @InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository<GeodataPlacesEntity>,
     @Inject(ILoggerRepository) private logger: ILoggerRepository,
+    @Inject(IConfigRepository) configRepository: IConfigRepository,
   ) {
+    this.vectorExtension = configRepository.getEnv().database.vectorExtension;
     this.logger.setContext(SearchRepository.name);
     this.faceColumns = this.assetFaceRepository.manager.connection
       .getMetadata(AssetFaceEntity)
@@ -440,7 +443,7 @@ export class SearchRepository implements ISearchRepository {
   }
 
   private getRuntimeConfig(numResults?: number): string {
-    if (getVectorExtension() === DatabaseExtension.VECTOR) {
+    if (this.vectorExtension === DatabaseExtension.VECTOR) {
       return 'SET LOCAL hnsw.ef_search = 1000;'; // mitigate post-filter recall
     }
 

diff --git a/server/src/services/database.service.spec.ts b/server/src/services/database.service.spec.ts
@@ -57,7 +57,17 @@ describe(DatabaseService.name, () => {
   ])('should work with $extensionName', ({ extension, extensionName }) => {
     beforeEach(() => {
       configMock.getEnv.mockReturnValue(
-        mockEnvData({ database: { skipMigrations: false, vectorExtension: extension } }),
+        mockEnvData({
+          database: {
+            host: 'database',
+            port: 5432,
+            username: 'postgres',
+            password: 'postgres',
+            name: 'immich',
+            skipMigrations: false,
+            vectorExtension: extension,
+          },
+        }),
       );
     });
 
@@ -245,6 +255,11 @@ describe(DatabaseService.name, () => {
     configMock.getEnv.mockReturnValue(
       mockEnvData({
         database: {
+          host: 'database',
+          port: 5432,
+          username: 'postgres',
+          password: 'postgres',
+          name: 'immich',
           skipMigrations: true,
           vectorExtension: DatabaseExtension.VECTORS,
         },
@@ -260,6 +275,11 @@ describe(DatabaseService.name, () => {
     configMock.getEnv.mockReturnValue(
       mockEnvData({
         database: {
+          host: 'database',
+          port: 5432,
+          username: 'postgres',
+          password: 'postgres',
+          name: 'immich',
           skipMigrations: true,
           vectorExtension: DatabaseExtension.VECTOR,
         },

diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts
@@ -10,6 +10,12 @@ const envData: EnvData = {
   buildMetadata: {},
 
   database: {
+    host: 'database',
+    port: 5432,
+    username: 'postgres',
+    password: 'postgres',
+    name: 'immich',
+
     skipMigrations: false,
     vectorExtension: DatabaseExtension.VECTORS,
   },