Merge remote-tracking branch 'upstream/main' into fix-back-scroll

cli/src/commands/asset.ts

@@ -1,5 +1,6 @@
 import {
   Action,
+  AssetBulkUploadCheckItem,
   AssetBulkUploadCheckResult,
   AssetMediaResponseDto,
   AssetMediaStatus,
@@ -11,7 +12,7 @@ import {
   getSupportedMediaTypes,
 } from '@immich/sdk';
 import byteSize from 'byte-size';
-import { Presets, SingleBar } from 'cli-progress';
+import { MultiBar, Presets, SingleBar } from 'cli-progress';
 import { chunk } from 'lodash-es';
 import { Stats, createReadStream } from 'node:fs';
 import { stat, unlink } from 'node:fs/promises';
@@ -90,23 +91,23 @@ export const checkForDuplicates = async (files: string[], { concurrency, skipHas
     return { newFiles: files, duplicates: [] };
   }
 
-  const progressBar = new SingleBar(
-    { format: 'Checking files | {bar} | {percentage}% | ETA: {eta}s | {value}/{total} assets' },
+  const multiBar = new MultiBar(
+    { format: '{message} | {bar} | {percentage}% | ETA: {eta}s | {value}/{total} assets' },
     Presets.shades_classic,
   );
 
-  progressBar.start(files.length, 0);
+  const hashProgressBar = multiBar.create(files.length, 0, { message: 'Hashing files          ' });
+  const checkProgressBar = multiBar.create(files.length, 0, { message: 'Checking for duplicates' });
 
   const newFiles: string[] = [];
   const duplicates: Asset[] = [];
 
-  const queue = new Queue<string[], AssetBulkUploadCheckResults>(
-    async (filepaths: string[]) => {
-      const dto = await Promise.all(
-        filepaths.map(async (filepath) => ({ id: filepath, checksum: await sha1(filepath) })),
-      );
-      const response = await checkBulkUpload({ assetBulkUploadCheckDto: { assets: dto } });
+  const checkBulkUploadQueue = new Queue<AssetBulkUploadCheckItem[], void>(
+    async (assets: AssetBulkUploadCheckItem[]) => {
+      const response = await checkBulkUpload({ assetBulkUploadCheckDto: { assets } });
+
       const results = response.results as AssetBulkUploadCheckResults;
+
       for (const { id: filepath, assetId, action } of results) {
         if (action === Action.Accept) {
           newFiles.push(filepath);
@@ -115,19 +116,46 @@ export const checkForDuplicates = async (files: string[], { concurrency, skipHas
           duplicates.push({ id: assetId as string, filepath });
         }
       }
-      progressBar.increment(filepaths.length);
+
+      checkProgressBar.increment(assets.length);
+    },
+    { concurrency, retry: 3 },
+  );
+
+  const results: { id: string; checksum: string }[] = [];
+  let checkBulkUploadRequests: AssetBulkUploadCheckItem[] = [];
+
+  const queue = new Queue<string, AssetBulkUploadCheckItem[]>(
+    async (filepath: string): Promise<AssetBulkUploadCheckItem[]> => {
+      const dto = { id: filepath, checksum: await sha1(filepath) };
+
+      results.push(dto);
+      checkBulkUploadRequests.push(dto);
+      if (checkBulkUploadRequests.length === 5000) {
+        const batch = checkBulkUploadRequests;
+        checkBulkUploadRequests = [];
+        void checkBulkUploadQueue.push(batch);
+      }
+
+      hashProgressBar.increment();
       return results;
     },
     { concurrency, retry: 3 },
   );
 
-  for (const items of chunk(files, concurrency)) {
-    await queue.push(items);
+  for (const item of files) {
+    void queue.push(item);
   }
 
   await queue.drained();
 
-  progressBar.stop();
+  if (checkBulkUploadRequests.length > 0) {
+    void checkBulkUploadQueue.push(checkBulkUploadRequests);
+  }
+
+  await checkBulkUploadQueue.drained();
+
+  multiBar.stop();
 
   console.log(`Found ${newFiles.length} new files and ${duplicates.length} duplicate${s(duplicates.length)}`);
 
@@ -201,8 +229,8 @@ export const uploadFiles = async (files: string[], { dryRun, concurrency }: Uplo
     { concurrency, retry: 3 },
   );
 
-  for (const filepath of files) {
-    await queue.push(filepath);
+  for (const item of files) {
+    void queue.push(item);
   }
 
   await queue.drained();

cli/src/queue.ts

@@ -72,8 +72,8 @@ export class Queue<T, R> {
    * @returns Promise<void> - The returned Promise will be resolved when all tasks in the queue have been processed by a worker.
    * This promise could be ignored as it will not lead to a `unhandledRejection`.
    */
-  async drained(): Promise<void> {
-    await this.queue.drain();
+  drained(): Promise<void> {
+    return this.queue.drained();
   }
 
   /**

docker/docker-compose.prod.yml

@@ -94,7 +94,7 @@ services:
     container_name: immich_prometheus
     ports:
       - 9090:9090
-    image: prom/prometheus@sha256:2659f4c2ebb718e7695cb9b25ffa7d6be64db013daba13e05c875451cf51b0d3
+    image: prom/prometheus@sha256:3b9b2a15d376334da8c286d995777d3b9315aa666d2311170ada6059a517b74f
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
       - prometheus-data:/prometheus

docs/docs/guides/database-queries.md

@@ -98,6 +98,10 @@ SELECT * FROM "move_history";
 SELECT * FROM "users";
 ```
 
+```sql title="Get owner info from asset ID"
+SELECT "users".* FROM "users" JOIN "assets" ON "users"."id" = "assets"."ownerId" WHERE "assets"."id" = 'fa310b01-2f26-4b7a-9042-d578226e021f';
+```
+
 ## System Config
 
 ```sql title="Custom settings"

e2e/src/api/specs/server.e2e-spec.ts

@@ -163,11 +163,15 @@ describe('/server', () => {
       expect(body).toEqual({
         photos: 0,
         usage: 0,
+        usagePhotos: 0,
+        usageVideos: 0,
         usageByUser: [
           {
             quotaSizeInBytes: null,
             photos: 0,
             usage: 0,
+            usagePhotos: 0,
+            usageVideos: 0,
             userName: 'Immich Admin',
             userId: admin.userId,
             videos: 0,
@@ -176,6 +180,8 @@ describe('/server', () => {
             quotaSizeInBytes: null,
             photos: 0,
             usage: 0,
+            usagePhotos: 0,
+            usageVideos: 0,
             userName: 'User 1',
             userId: nonAdmin.userId,
             videos: 0,

e2e/src/cli/specs/upload.e2e-spec.ts

@@ -103,7 +103,7 @@ describe(`immich upload`, () => {
   describe(`immich upload /path/to/file.jpg`, () => {
     it('should upload a single file', async () => {
       const { stderr, stdout, exitCode } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(stdout.split('\n')).toEqual(
         expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]),
       );
@@ -126,7 +126,7 @@ describe(`immich upload`, () => {
           const expectedCount = Object.entries(files).filter((entry) => entry[1]).length;
 
           const { stderr, stdout, exitCode } = await immichCli(['upload', ...commandLine]);
-          expect(stderr).toBe('');
+          expect(stderr).toContain('{message}');
           expect(stdout.split('\n')).toEqual(
             expect.arrayContaining([expect.stringContaining(`Successfully uploaded ${expectedCount} new asset`)]),
           );
@@ -154,7 +154,7 @@ describe(`immich upload`, () => {
       cpSync(`${testAssetDir}/albums/nature/silver_fir.jpg`, testPaths[1]);
 
       const { stderr, stdout, exitCode } = await immichCli(['upload', ...testPaths]);
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(stdout.split('\n')).toEqual(
         expect.arrayContaining([expect.stringContaining('Successfully uploaded 2 new assets')]),
       );
@@ -169,7 +169,7 @@ describe(`immich upload`, () => {
 
     it('should skip a duplicate file', async () => {
       const first = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
-      expect(first.stderr).toBe('');
+      expect(first.stderr).toContain('{message}');
       expect(first.stdout.split('\n')).toEqual(
         expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]),
       );
@@ -179,7 +179,7 @@ describe(`immich upload`, () => {
       expect(assets.total).toBe(1);
 
       const second = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
-      expect(second.stderr).toBe('');
+      expect(second.stderr).toContain('{message}');
       expect(second.stdout.split('\n')).toEqual(
         expect.arrayContaining([
           expect.stringContaining('Found 0 new files and 1 duplicate'),
@@ -205,7 +205,7 @@ describe(`immich upload`, () => {
         `${testAssetDir}/albums/nature/silver_fir.jpg`,
         '--dry-run',
       ]);
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(stdout.split('\n')).toEqual(
         expect.arrayContaining([expect.stringContaining('Would have uploaded 1 asset')]),
       );
@@ -217,7 +217,7 @@ describe(`immich upload`, () => {
 
     it('dry run should handle duplicates', async () => {
       const first = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
-      expect(first.stderr).toBe('');
+      expect(first.stderr).toContain('{message}');
       expect(first.stdout.split('\n')).toEqual(
         expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]),
       );
@@ -227,7 +227,7 @@ describe(`immich upload`, () => {
       expect(assets.total).toBe(1);
 
       const second = await immichCli(['upload', `${testAssetDir}/albums/nature/`, '--dry-run']);
-      expect(second.stderr).toBe('');
+      expect(second.stderr).toContain('{message}');
       expect(second.stdout.split('\n')).toEqual(
         expect.arrayContaining([
           expect.stringContaining('Found 8 new files and 1 duplicate'),
@@ -241,7 +241,7 @@ describe(`immich upload`, () => {
   describe('immich upload --recursive', () => {
     it('should upload a folder recursively', async () => {
       const { stderr, stdout, exitCode } = await immichCli(['upload', `${testAssetDir}/albums/nature/`, '--recursive']);
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(stdout.split('\n')).toEqual(
         expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 new assets')]),
       );
@@ -267,7 +267,7 @@ describe(`immich upload`, () => {
           expect.stringContaining('Successfully updated 9 assets'),
         ]),
       );
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(exitCode).toBe(0);
 
       const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) });
@@ -283,7 +283,7 @@ describe(`immich upload`, () => {
       expect(response1.stdout.split('\n')).toEqual(
         expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 new assets')]),
       );
-      expect(response1.stderr).toBe('');
+      expect(response1.stderr).toContain('{message}');
       expect(response1.exitCode).toBe(0);
 
       const assets1 = await getAssetStatistics({}, { headers: asKeyAuth(key) });
@@ -299,7 +299,7 @@ describe(`immich upload`, () => {
           expect.stringContaining('Successfully updated 9 assets'),
         ]),
       );
-      expect(response2.stderr).toBe('');
+      expect(response2.stderr).toContain('{message}');
       expect(response2.exitCode).toBe(0);
 
       const assets2 = await getAssetStatistics({}, { headers: asKeyAuth(key) });
@@ -325,7 +325,7 @@ describe(`immich upload`, () => {
           expect.stringContaining('Would have updated albums of 9 assets'),
         ]),
       );
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(exitCode).toBe(0);
 
       const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) });
@@ -351,7 +351,7 @@ describe(`immich upload`, () => {
           expect.stringContaining('Successfully updated 9 assets'),
         ]),
       );
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(exitCode).toBe(0);
 
       const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) });
@@ -377,7 +377,7 @@ describe(`immich upload`, () => {
           expect.stringContaining('Would have updated albums of 9 assets'),
         ]),
       );
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(exitCode).toBe(0);
 
       const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) });
@@ -408,7 +408,7 @@ describe(`immich upload`, () => {
           expect.stringContaining('Deleting assets that have been uploaded'),
         ]),
       );
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(exitCode).toBe(0);
 
       const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) });
@@ -434,7 +434,7 @@ describe(`immich upload`, () => {
           expect.stringContaining('Would have deleted 9 local assets'),
         ]),
       );
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(exitCode).toBe(0);
 
       const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) });
@@ -493,7 +493,7 @@ describe(`immich upload`, () => {
         '2',
       ]);
 
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(stdout.split('\n')).toEqual(
         expect.arrayContaining([
           'Found 9 new files and 0 duplicates',
@@ -534,7 +534,7 @@ describe(`immich upload`, () => {
         'silver_fir.jpg',
       ]);
 
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(stdout.split('\n')).toEqual(
         expect.arrayContaining([
           'Found 8 new files and 0 duplicates',
@@ -555,7 +555,7 @@ describe(`immich upload`, () => {
         '!(*_*_*).jpg',
       ]);
 
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(stdout.split('\n')).toEqual(
         expect.arrayContaining([
           'Found 1 new files and 0 duplicates',
@@ -577,7 +577,7 @@ describe(`immich upload`, () => {
         '--dry-run',
       ]);
 
-      expect(stderr).toBe('');
+      expect(stderr).toContain('{message}');
       expect(stdout.split('\n')).toEqual(
         expect.arrayContaining([
           'Found 8 new files and 0 duplicates',

mobile/android/app/proguard-rules.pro

@@ -0,0 +1,32 @@
+##---------------Begin: proguard configuration for Gson  ----------
+# Gson uses generic type information stored in a class file when working with fields. Proguard
+# removes such information by default, so configure it to keep all of it.
+-keepattributes Signature
+
+# For using GSON @Expose annotation
+-keepattributes *Annotation*
+
+# Gson specific classes
+-dontwarn sun.misc.**
+#-keep class com.google.gson.stream.** { *; }
+
+# Application classes that will be serialized/deserialized over Gson
+-keep class com.google.gson.examples.android.model.** { <fields>; }
+
+# Prevent proguard from stripping interface information from TypeAdapter, TypeAdapterFactory,
+# JsonSerializer, JsonDeserializer instances (so they can be used in @JsonAdapter)
+-keep class * extends com.google.gson.TypeAdapter
+-keep class * implements com.google.gson.TypeAdapterFactory
+-keep class * implements com.google.gson.JsonSerializer
+-keep class * implements com.google.gson.JsonDeserializer
+
+# Prevent R8 from leaving Data object members always null
+-keepclassmembers,allowobfuscation class * {
+  @com.google.gson.annotations.SerializedName <fields>;
+}
+
+# Retain generic signatures of TypeToken and its subclasses with R8 version 3.0 and higher.
+-keep,allowobfuscation,allowshrinking class com.google.gson.reflect.TypeToken
+-keep,allowobfuscation,allowshrinking class * extends com.google.gson.reflect.TypeToken
+
+##---------------End: proguard configuration for Gson  ----------

mobile/android/gradle.properties

@@ -2,4 +2,4 @@ org.gradle.jvmargs=-Xmx4096M
 android.useAndroidX=true
 android.enableJetifier=true
 android.nonTransitiveRClass=false
-android.nonFinalResIds=false
+android.nonFinalResIds=false