Skip to content

Commit

Permalink
fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
mertalev committed Feb 10, 2024
1 parent 4ff4b38 commit b2828a9
Show file tree
Hide file tree
Showing 8 changed files with 72 additions and 146 deletions.
2 changes: 1 addition & 1 deletion server/src/domain/audit/audit.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ export class AuditService {
`Found ${libraryFiles.size} original files, ${thumbFiles.size} thumbnails, ${videoFiles.size} encoded videos, ${profileFiles.size} profile files`,
);
const pagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (options) =>
this.assetRepository.getAll(options, { withDeleted: true }),
this.assetRepository.getAll(options, { status: { withDeleted: true } }),
);

let assetCount = 0;
Expand Down
2 changes: 1 addition & 1 deletion server/src/domain/download/download.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ export class DownloadService {
const userId = dto.userId;
await this.access.requirePermission(auth, Permission.TIMELINE_DOWNLOAD, userId);
return usePagination(PAGINATION_SIZE, (pagination) =>
this.assetRepository.getByUserId(pagination, userId, { isVisible: true }),
this.assetRepository.getByUserId(pagination, userId, { status: { isVisible: true } }),
);
}

Expand Down
2 changes: 1 addition & 1 deletion server/src/domain/media/media.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ export class MediaService {

const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination, { type: AssetType.VIDEO })
? this.assetRepository.getAll(pagination, { status: { type: AssetType.VIDEO } })
: this.assetRepository.getWithout(pagination, WithoutProperty.ENCODED_VIDEO);
});

Expand Down
4 changes: 2 additions & 2 deletions server/src/domain/trash/trash.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ export class TrashService {

async restore(auth: AuthDto): Promise<void> {
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.assetRepository.getByUserId(pagination, auth.user.id, { trashedBefore: DateTime.now().toJSDate() }),
this.assetRepository.getByUserId(pagination, auth.user.id, { date: { trashedBefore: DateTime.now().toJSDate() } }),
);

for await (const assets of assetPagination) {
Expand All @@ -44,7 +44,7 @@ export class TrashService {

async empty(auth: AuthDto): Promise<void> {
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.assetRepository.getByUserId(pagination, auth.user.id, { trashedBefore: DateTime.now().toJSDate() }),
this.assetRepository.getByUserId(pagination, auth.user.id, { date: { trashedBefore: DateTime.now().toJSDate() } }),
);

for await (const assets of assetPagination) {
Expand Down
4 changes: 1 addition & 3 deletions server/src/infra/infra.utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,10 +124,8 @@ export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {

export function searchAssetBuilder(
builder: SelectQueryBuilder<AssetEntity>,
options: AssetSearchBuilderOptions,
{ date, id, exif, path, relation, status }: AssetSearchBuilderOptions,
): SelectQueryBuilder<AssetEntity> {
const { date, id, exif, path, relation, status } = options;

if (date) {
builder.andWhere(
_.omitBy(
Expand Down
32 changes: 23 additions & 9 deletions server/src/infra/repositories/search.repository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import _ from 'lodash';
import { Repository } from 'typeorm';
import { vectorExt } from '../database.config';
import { DummyValue, GenerateSql } from '../infra.util';
import { asVector, isValidInteger, paginatedBuilder, searchAssetBuilder } from '../infra.utils';
import { vectorExt } from '../database.config';

@Injectable()
export class SearchRepository implements ISearchRepository {
Expand Down Expand Up @@ -70,23 +70,37 @@ export class SearchRepository implements ISearchRepository {
}

@GenerateSql({
params: [{ userIds: [DummyValue.UUID], embedding: Array.from({ length: 512 }, Math.random), numResults: 100 }],
params: [
{
pagination: { page: 0, size: 100 },
options: {
date: { takenAfter: DummyValue.DATE },
embedding: Array.from({ length: 512 }, Math.random),
exif: { cameraModel: DummyValue.STRING },
relation: { withStacked: true },
status: { isFavorite: true },
userIds: [DummyValue.UUID],
},
},
],
})
async searchCLIP(pagination: SearchPaginationOptions, options: SmartSearchOptions): Paginated<AssetEntity> {
async searchCLIP(
pagination: SearchPaginationOptions,
{ embedding, userIds, ...options }: SmartSearchOptions,
): Paginated<AssetEntity> {
let results: PaginationResult<AssetEntity> = { items: [], hasNextPage: false };

await this.assetRepository.manager.transaction(async (manager) => {
await manager.query(`SET LOCAL vectors.search_mode=vbase`);
let builder = manager.createQueryBuilder(AssetEntity, 'asset');
builder = searchAssetBuilder(builder, options);
builder
.innerJoin('a.smartSearch', 's')
.andWhere('a.ownerId IN (:...userIds )')
.orderBy('s.embedding <=> :embedding')
.setParameters({ userIds: options.userIds, embedding: asVector(options.embedding) });
.innerJoin('asset.smartSearch', 'search')
.andWhere('asset.ownerId IN (:...userIds )')
.orderBy('search.embedding <=> :embedding')
.setParameters({ userIds, embedding: asVector(embedding) });

await manager.query(this.getRuntimeConfig(pagination.size));
return paginatedBuilder<AssetEntity>(builder, {
results = await paginatedBuilder<AssetEntity>(builder, {
mode: PaginationMode.LIMIT_OFFSET,
skip: pagination.page * pagination.size,
take: pagination.size,
Expand Down
43 changes: 43 additions & 0 deletions server/src/infra/sql/search.repository.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
-- NOTE: This file is auto generated by ./sql-generator

-- SearchRepository.searchFaces
START TRANSACTION
SET
LOCAL vectors.enable_prefilter = on;

SET
LOCAL vectors.search_mode = basic;

SET
LOCAL vectors.hnsw_ef_search = 100
WITH
"cte" AS (
SELECT
"faces"."id" AS "id",
"faces"."assetId" AS "assetId",
"faces"."personId" AS "personId",
"faces"."imageWidth" AS "imageWidth",
"faces"."imageHeight" AS "imageHeight",
"faces"."boundingBoxX1" AS "boundingBoxX1",
"faces"."boundingBoxY1" AS "boundingBoxY1",
"faces"."boundingBoxX2" AS "boundingBoxX2",
"faces"."boundingBoxY2" AS "boundingBoxY2",
"faces"."embedding" <= > $1 AS "distance"
FROM
"asset_faces" "faces"
INNER JOIN "assets" "asset" ON "asset"."id" = "faces"."assetId"
AND ("asset"."deletedAt" IS NULL)
WHERE
"asset"."ownerId" IN ($2)
ORDER BY
"faces"."embedding" <= > $1 ASC
LIMIT
100
)
SELECT
res.*
FROM
"cte" "res"
WHERE
res.distance <= $3
COMMIT
129 changes: 0 additions & 129 deletions server/src/infra/sql/smart.info.repository.sql

This file was deleted.

0 comments on commit b2828a9

Please sign in to comment.