Merge pull request #12719 from nextcloud/autoupload_only_check_changes

Try to only process changed files for auto upload

app/src/androidTest/java/com/nextcloud/client/jobs/BackgroundJobManagerTest.kt

@@ -214,7 +214,7 @@ class BackgroundJobManagerTest {
         fun job_is_unique_and_replaces_previous_job() {
             verify(workManager).enqueueUniqueWork(
                 eq(BackgroundJobManagerImpl.JOB_CONTENT_OBSERVER),
-                eq(ExistingWorkPolicy.REPLACE),
+                eq(ExistingWorkPolicy.APPEND),
                 argThat(IsOneTimeWorkRequest())
             )
         }

app/src/main/java/com/nextcloud/client/jobs/BackgroundJobManager.kt

@@ -133,7 +133,12 @@ interface BackgroundJobManager {
     fun startImmediateFilesExportJob(files: Collection<OCFile>): LiveData<JobInfo?>
 
     fun schedulePeriodicFilesSyncJob()
-    fun startImmediateFilesSyncJob(skipCustomFolders: Boolean = false, overridePowerSaving: Boolean = false)
+
+    fun startImmediateFilesSyncJob(
+        overridePowerSaving: Boolean = false,
+        changedFiles: Array<String> = arrayOf<String>()
+    )
+
     fun scheduleOfflineSync()
 
     fun scheduleMediaFoldersDetectionJob()

app/src/main/java/com/nextcloud/client/jobs/BackgroundJobManagerImpl.kt

@@ -98,7 +98,7 @@ internal class BackgroundJobManagerImpl(
 
         const val JOB_TEST = "test_job"
 
-        const val MAX_CONTENT_TRIGGER_DELAY_MS = 1500L
+        const val MAX_CONTENT_TRIGGER_DELAY_MS = 10000L
 
         const val TAG_PREFIX_NAME = "name"
         const val TAG_PREFIX_USER = "user"
@@ -277,7 +277,7 @@ internal class BackgroundJobManagerImpl(
             .setConstraints(constrains)
             .build()
 
-        workManager.enqueueUniqueWork(JOB_CONTENT_OBSERVER, ExistingWorkPolicy.REPLACE, request)
+        workManager.enqueueUniqueWork(JOB_CONTENT_OBSERVER, ExistingWorkPolicy.APPEND, request)
     }
 
     override fun schedulePeriodicContactsBackup(user: User) {
@@ -425,10 +425,13 @@ internal class BackgroundJobManagerImpl(
         workManager.enqueueUniquePeriodicWork(JOB_PERIODIC_FILES_SYNC, ExistingPeriodicWorkPolicy.REPLACE, request)
     }
 
-    override fun startImmediateFilesSyncJob(skipCustomFolders: Boolean, overridePowerSaving: Boolean) {
+    override fun startImmediateFilesSyncJob(
+        overridePowerSaving: Boolean,
+        changedFiles: Array<String>
+    ) {
         val arguments = Data.Builder()
-            .putBoolean(FilesSyncWork.SKIP_CUSTOM, skipCustomFolders)
             .putBoolean(FilesSyncWork.OVERRIDE_POWER_SAVING, overridePowerSaving)
+            .putStringArray(FilesSyncWork.CHANGED_FILES, changedFiles)
             .build()
 
         val request = oneTimeRequestBuilder(
@@ -438,7 +441,7 @@ internal class BackgroundJobManagerImpl(
             .setInputData(arguments)
             .build()
 
-        workManager.enqueueUniqueWork(JOB_IMMEDIATE_FILES_SYNC, ExistingWorkPolicy.KEEP, request)
+        workManager.enqueueUniqueWork(JOB_IMMEDIATE_FILES_SYNC, ExistingWorkPolicy.APPEND, request)
     }
 
     override fun scheduleOfflineSync() {

app/src/main/java/com/nextcloud/client/jobs/ContentObserverWork.kt

@@ -61,7 +61,11 @@ class ContentObserverWork(
     private fun checkAndStartFileSyncJob() {
         val syncFolders = syncerFolderProvider.countEnabledSyncedFolders() > 0
         if (!powerManagementService.isPowerSavingEnabled && syncFolders) {
-            backgroundJobManager.startImmediateFilesSyncJob(true, false)
+            val changedFiles = mutableListOf<String>()
+            for (uri in params.triggeredContentUris) {
+                changedFiles.add(uri.toString())
+            }
+            backgroundJobManager.startImmediateFilesSyncJob(false, changedFiles.toTypedArray())
         }
     }
 

app/src/main/java/com/nextcloud/client/jobs/FilesSyncWork.kt

@@ -29,8 +29,8 @@ import android.os.Build
 import android.text.TextUtils
 import androidx.core.app.NotificationCompat
 import androidx.exifinterface.media.ExifInterface
-import androidx.work.CoroutineWorker
 import androidx.work.ForegroundInfo
+import androidx.work.Worker
 import androidx.work.WorkerParameters
 import com.nextcloud.client.account.UserAccountManager
 import com.nextcloud.client.device.PowerManagementService
@@ -72,19 +72,23 @@ class FilesSyncWork(
     private val powerManagementService: PowerManagementService,
     private val syncedFolderProvider: SyncedFolderProvider,
     private val backgroundJobManager: BackgroundJobManager
-) : CoroutineWorker(context, params) {
+) : Worker(context, params) {
 
     companion object {
         const val TAG = "FilesSyncJob"
         const val SKIP_CUSTOM = "skipCustom"
         const val OVERRIDE_POWER_SAVING = "overridePowerSaving"
+        const val CHANGED_FILES = "changedFiles"
         const val FOREGROUND_SERVICE_ID = 414
     }
 
     @Suppress("MagicNumber")
-    private fun createForegroundInfo(progressPercent: Int): ForegroundInfo {
-        // update throughout worker execution to give use feedback how far worker is
+    private fun updateForegroundWorker(progressPercent: Int, useForegroundWorker: Boolean) {
+        if (useForegroundWorker) {
+            return
+        }
 
+        // update throughout worker execution to give use feedback how far worker is
         val notification = NotificationCompat.Builder(context, NotificationUtils.NOTIFICATION_CHANNEL_FILE_SYNC)
             .setTicker(context.getString(R.string.autoupload_worker_foreground_info))
             .setContentText(context.getString(R.string.autoupload_worker_foreground_info))
@@ -93,17 +97,18 @@ class FilesSyncWork(
             .setOngoing(true)
             .setProgress(100, progressPercent, false)
             .build()
-        return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+        val foregroundInfo = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
             ForegroundInfo(FOREGROUND_SERVICE_ID, notification, ForegroundServiceType.DataSync.getId())
         } else {
             ForegroundInfo(FOREGROUND_SERVICE_ID, notification)
         }
+
+        setForegroundAsync(foregroundInfo)
     }
 
     @Suppress("MagicNumber")
-    override suspend fun doWork(): Result {
+    override fun doWork(): Result {
         backgroundJobManager.logStartOfWorker(BackgroundJobManagerImpl.formatClassTag(this::class))
-        setForeground(createForegroundInfo(0))
 
         val overridePowerSaving = inputData.getBoolean(OVERRIDE_POWER_SAVING, false)
         // If we are in power save mode, better to postpone upload
@@ -114,26 +119,35 @@ class FilesSyncWork(
         }
         val resources = context.resources
         val lightVersion = resources.getBoolean(R.bool.syncedFolder_light)
-        val skipCustom = inputData.getBoolean(SKIP_CUSTOM, false)
         FilesSyncHelper.restartJobsIfNeeded(
             uploadsStorageManager,
             userAccountManager,
             connectivityService,
             powerManagementService
         )
-        setForeground(createForegroundInfo(5))
-        FilesSyncHelper.insertAllDBEntries(skipCustom, syncedFolderProvider)
-        setForeground(createForegroundInfo(50))
+
+        // Get changed files from ContentObserverWork (only images and videos) or by scanning filesystem
+        val changedFiles = inputData.getStringArray(CHANGED_FILES)
+        collectChangedFiles(changedFiles)
+
         // Create all the providers we'll need
         val filesystemDataProvider = FilesystemDataProvider(contentResolver)
         val currentLocale = resources.configuration.locale
         val dateFormat = SimpleDateFormat("yyyy:MM:dd HH:mm:ss", currentLocale)
         dateFormat.timeZone = TimeZone.getTimeZone(TimeZone.getDefault().id)
 
+        // start upload of changed / new files
         val syncedFolders = syncedFolderProvider.syncedFolders
         for ((index, syncedFolder) in syncedFolders.withIndex()) {
-            setForeground(createForegroundInfo((50 + (index.toDouble() / syncedFolders.size.toDouble()) * 50).toInt()))
-            if (syncedFolder.isEnabled && (!skipCustom || MediaFolderType.CUSTOM != syncedFolder.type)) {
+            updateForegroundWorker(
+                (50 + (index.toDouble() / syncedFolders.size.toDouble()) * 50).toInt(),
+                changedFiles.isNullOrEmpty()
+            )
+            if (syncedFolder.isEnabled && (
+                    changedFiles.isNullOrEmpty() ||
+                        MediaFolderType.CUSTOM != syncedFolder.type
+                    )
+            ) {
                 syncFolder(
                     context,
                     resources,
@@ -150,6 +164,19 @@ class FilesSyncWork(
         return result
     }
 
+    @Suppress("MagicNumber")
+    private fun collectChangedFiles(changedFiles: Array<String>?) {
+        if (!changedFiles.isNullOrEmpty()) {
+            FilesSyncHelper.insertChangedEntries(syncedFolderProvider, changedFiles)
+        } else {
+            // Check every file in every synced folder for changes and update
+            // filesystemDataProvider database (potentially needs a long time so use foreground worker)
+            updateForegroundWorker(5, true)
+            FilesSyncHelper.insertAllDBEntries(syncedFolderProvider)
+            updateForegroundWorker(50, true)
+        }
+    }
+
     @Suppress("LongMethod") // legacy code
     private fun syncFolder(
         context: Context,

app/src/main/java/com/owncloud/android/MainApp.java

@@ -556,7 +556,7 @@ public static void initSyncOperations(
         }
 
         if (!preferences.isAutoUploadInitialized()) {
-            backgroundJobManager.startImmediateFilesSyncJob(false, false);
+            backgroundJobManager.startImmediateFilesSyncJob(false, new String[]{});
             preferences.setAutoUploadInit(true);
         }
 

app/src/main/java/com/owncloud/android/datamodel/SyncedFolder.java

@@ -278,4 +278,8 @@ public boolean isExcludeHidden() {
     public void setExcludeHidden(boolean excludeHidden) {
         this.excludeHidden = excludeHidden;
     }
+
+    public boolean containsFile(String filePath){
+        return filePath.contains(localPath);
+    }
 }

app/src/main/java/com/owncloud/android/ui/activity/SyncedFoldersActivity.kt

@@ -40,7 +40,6 @@ import androidx.recyclerview.widget.GridLayoutManager
 import com.nextcloud.client.core.Clock
 import com.nextcloud.client.device.PowerManagementService
 import com.nextcloud.client.di.Injectable
-import com.nextcloud.client.jobs.BackgroundJobManager
 import com.nextcloud.client.jobs.MediaFoldersDetectionWork
 import com.nextcloud.client.jobs.NotificationWork
 import com.nextcloud.client.jobs.upload.FileUploadWorker
@@ -156,9 +155,6 @@ class SyncedFoldersActivity :
     @Inject
     lateinit var clock: Clock
 
-    @Inject
-    lateinit var backgroundJobManager: BackgroundJobManager
-
     @Inject
     lateinit var viewThemeUtils: ViewThemeUtils
 
@@ -584,7 +580,7 @@ class SyncedFoldersActivity :
             }
         }
         if (syncedFolderDisplayItem.isEnabled) {
-            backgroundJobManager.startImmediateFilesSyncJob(skipCustomFolders = false, overridePowerSaving = false)
+            backgroundJobManager.startImmediateFilesSyncJob(overridePowerSaving = false)
             showBatteryOptimizationInfo()
         }
     }
@@ -714,7 +710,7 @@ class SyncedFoldersActivity :
             // existing synced folder setup to be updated
             syncedFolderProvider.updateSyncFolder(item)
             if (item.isEnabled) {
-                backgroundJobManager.startImmediateFilesSyncJob(skipCustomFolders = false, overridePowerSaving = false)
+                backgroundJobManager.startImmediateFilesSyncJob(overridePowerSaving = false)
             } else {
                 val syncedFolderInitiatedKey = KEY_SYNCED_FOLDER_INITIATED_PREFIX + item.id
                 val arbitraryDataProvider =
@@ -731,7 +727,7 @@ class SyncedFoldersActivity :
         if (storedId != -1L) {
             item.id = storedId
             if (item.isEnabled) {
-                backgroundJobManager.startImmediateFilesSyncJob(skipCustomFolders = false, overridePowerSaving = false)
+                backgroundJobManager.startImmediateFilesSyncJob(overridePowerSaving = false)
             } else {
                 val syncedFolderInitiatedKey = KEY_SYNCED_FOLDER_INITIATED_PREFIX + item.id
                 arbitraryDataProvider.deleteKeyForAccount("global", syncedFolderInitiatedKey)

app/src/main/java/com/owncloud/android/ui/activity/UploadListActivity.java

@@ -208,7 +208,7 @@ private void loadItems() {
     }
 
     private void refresh() {
-        backgroundJobManager.startImmediateFilesSyncJob(false, true);
+        backgroundJobManager.startImmediateFilesSyncJob(true,new String[]{});
 
         if (uploadsStorageManager.getFailedUploads().length > 0) {
             new Thread(() -> {

app/src/main/java/com/owncloud/android/utils/FilesSyncHelper.java

@@ -130,15 +130,51 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) {
         }
     }
 
-    public static void insertAllDBEntries(boolean skipCustom,
-                                          SyncedFolderProvider syncedFolderProvider) {
+    public static void insertAllDBEntries(SyncedFolderProvider syncedFolderProvider) {
         for (SyncedFolder syncedFolder : syncedFolderProvider.getSyncedFolders()) {
-            if (syncedFolder.isEnabled() && (!skipCustom || syncedFolder.getType() != MediaFolderType.CUSTOM)) {
+            if (syncedFolder.isEnabled()) {
                 insertAllDBEntriesForSyncedFolder(syncedFolder);
             }
         }
     }
 
+    public static void insertChangedEntries(SyncedFolderProvider syncedFolderProvider,
+                                            String[] changedFiles) {
+        final ContentResolver contentResolver = MainApp.getAppContext().getContentResolver();
+        final FilesystemDataProvider filesystemDataProvider = new FilesystemDataProvider(contentResolver);
+        for (String changedFileURI : changedFiles){
+            String changedFile = getFileFromURI(changedFileURI);
+            for (SyncedFolder syncedFolder : syncedFolderProvider.getSyncedFolders()) {
+                if (syncedFolder.isEnabled() && syncedFolder.containsFile(changedFile)){
+                    File file = new File(changedFile);
+                    filesystemDataProvider.storeOrUpdateFileValue(changedFile,
+                                                                  file.lastModified(),file.isDirectory(),
+                                                                  syncedFolder);
+                    break;
+                }
+            }
+        }
+    }
+
+    private static String getFileFromURI(String uri){
+        final Context context = MainApp.getAppContext();
+
+        Cursor cursor;
+        int column_index_data;
+        String filePath = null;
+
+        String[] projection = {MediaStore.MediaColumns.DATA};
+
+        cursor = context.getContentResolver().query(Uri.parse(uri), projection, null, null, null, null);
+
+        if (cursor != null && cursor.moveToFirst()) {
+            column_index_data = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns.DATA);
+            filePath = cursor.getString(column_index_data);
+            cursor.close();
+        }
+        return filePath;
+    }
+
     private static void insertContentIntoDB(Uri uri, SyncedFolder syncedFolder) {
         final Context context = MainApp.getAppContext();
         final ContentResolver contentResolver = context.getContentResolver();