Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make filesync worker persistant #13217

Closed
wants to merge 4 commits into from
Closed

Make filesync worker persistant #13217

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
034e9bc
remove old logic
JonasMayerDev May 22, 2024
34d025f
First prototype (Maybe not working yet)
JonasMayerDev Jun 27, 2024
2921bd9
Fix wrong db query
JonasMayerDev Jun 27, 2024
5dd0611
add logs for testing
JonasMayerDev Jul 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 75 additions & 71 deletions app/src/main/java/com/nextcloud/client/jobs/FilesSyncWork.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,14 @@ package com.nextcloud.client.jobs
import android.content.ContentResolver
import android.content.Context
import android.content.res.Resources
import android.database.Cursor
import android.text.TextUtils
import androidx.exifinterface.media.ExifInterface
import androidx.work.Worker
import androidx.work.WorkerParameters
import com.nextcloud.client.account.UserAccountManager
import com.nextcloud.client.device.PowerManagementService
import com.nextcloud.client.jobs.upload.FileUploadHelper
import com.nextcloud.client.jobs.upload.FileUploadHelper.Companion.instance
import com.nextcloud.client.jobs.upload.FileUploadWorker
import com.nextcloud.client.network.ConnectivityService
import com.nextcloud.client.preferences.SubFolderRule
Expand All @@ -35,7 +36,6 @@ import com.owncloud.android.ui.activity.SettingsActivity
import com.owncloud.android.utils.FileStorageUtils
import com.owncloud.android.utils.FilesSyncHelper
import com.owncloud.android.utils.MimeType
import com.owncloud.android.utils.MimeTypeUtil
import java.io.File
import java.text.ParsePosition
import java.text.SimpleDateFormat
Expand All @@ -60,15 +60,19 @@ class FilesSyncWork(
const val OVERRIDE_POWER_SAVING = "overridePowerSaving"
const val CHANGED_FILES = "changedFiles"
const val SYNCED_FOLDER_ID = "syncedFolderId"

var realScanIntervalMS = 1000L
}

private lateinit var syncedFolder: SyncedFolder
private var cursor: android.database.Cursor? = null

@Suppress("MagicNumber")
override fun doWork(): Result {
val syncFolderId = inputData.getLong(SYNCED_FOLDER_ID, -1)
val changedFiles = inputData.getStringArray(CHANGED_FILES)


backgroundJobManager.logStartOfWorker(BackgroundJobManagerImpl.formatClassTag(this::class) + "_" + syncFolderId)
Log_OC.d(TAG, "File-sync worker started for folder ID: $syncFolderId")

Expand All @@ -82,8 +86,19 @@ class FilesSyncWork(
return result
}

val resources = context.resources
val lightVersion = resources.getBoolean(R.bool.syncedFolder_light)
val filesystemDataProvider = FilesystemDataProvider(contentResolver)
var current = System.currentTimeMillis()
FilesSyncHelper.insertNewFilesIntoFSDatabase(syncedFolder.localPath, syncedFolder, filesystemDataProvider)
Log_OC.d(TAG, "File-sync worker (${syncedFolder.remotePath}) inserted new files into DB in " + (System.currentTimeMillis() - current) + "ms")
current = System.currentTimeMillis()
cursor = filesystemDataProvider.getFilesCursorAscOrderedForSyncedFolder(syncedFolder)
Log_OC.d(TAG, "File-sync worker (${syncedFolder.remotePath}) got files cursor in " + (System.currentTimeMillis() - current) + "ms")
current = System.currentTimeMillis()
checkFilesFormDBForChanges(syncedFolder, filesystemDataProvider, cursor)
Log_OC.d(TAG, "File-sync worker (${syncedFolder.remotePath}) checked files for changes in " + (System.currentTimeMillis() - current) + "ms")

// Todo: update syncedFolder.lastScanTimestampMs

FilesSyncHelper.restartUploadsIfNeeded(
uploadsStorageManager,
userAccountManager,
Expand All @@ -97,24 +112,8 @@ class FilesSyncWork(
"File-sync worker (${syncedFolder.remotePath}) changed files from observer: " +
changedFiles.contentToString()
)
collectChangedFiles(changedFiles)
Log_OC.d(TAG, "File-sync worker (${syncedFolder.remotePath}) finished checking files.")

// Create all the providers we'll need
val filesystemDataProvider = FilesystemDataProvider(contentResolver)
val currentLocale = resources.configuration.locale
val dateFormat = SimpleDateFormat("yyyy:MM:dd HH:mm:ss", currentLocale)
dateFormat.timeZone = TimeZone.getTimeZone(TimeZone.getDefault().id)

syncFolder(
context,
resources,
lightVersion,
filesystemDataProvider,
currentLocale,
dateFormat,
syncedFolder
)

Log_OC.d(TAG, "File-sync worker (${syncedFolder.remotePath}) finished")
val result = Result.success()
Expand All @@ -126,6 +125,11 @@ class FilesSyncWork(
return result
}

override fun onStopped() {
this.cursor?.close()
super.onStopped()
}

private fun setSyncedFolder(syncedFolderID: Long): Boolean {
val syncedFolderTmp = syncedFolderProvider.getSyncedFolderByID(syncedFolderID)
if (syncedFolderTmp == null || !syncedFolderTmp.isEnabled || !syncedFolderTmp.isExisting) {
Expand Down Expand Up @@ -164,6 +168,12 @@ class FilesSyncWork(
return true
}

if ((syncedFolder.lastScanTimestampMs + realScanIntervalMS) > System.currentTimeMillis()){
Log_OC.d(TAG, "File-sync kill worker since started before scan " +
"Interval and nothing todo (${syncedFolder.localPath})!")
return true
}

if (syncedFolder.isChargingOnly &&
!powerManagementService.battery.isCharging &&
!powerManagementService.battery.isFull
Expand All @@ -175,62 +185,58 @@ class FilesSyncWork(
return false
}

@Suppress("MagicNumber")
private fun collectChangedFiles(changedFiles: Array<String>?) {
if (!changedFiles.isNullOrEmpty()) {
FilesSyncHelper.insertChangedEntries(syncedFolder, changedFiles)
} else {
// Check every file in synced folder for changes and update
// filesystemDataProvider database (potentially needs a long time)
FilesSyncHelper.insertAllDBEntriesForSyncedFolder(syncedFolder)
private fun checkFilesFormDBForChanges(
syncedFolder: SyncedFolder,
filesystemDataProvider: FilesystemDataProvider,
cursor: Cursor?
) {
// TODO When completely done with folder set syncedFolder lastScanTimestampMs
if (cursor == null) {
return
}
if (cursor.moveToFirst()) {
do {
val fileData =
filesystemDataProvider.getFileSystemDataSetFromCursor(cursor, syncedFolder)

if ((fileData.foundAt + realScanIntervalMS)
> System.currentTimeMillis()
) {
break
}

if(FilesSyncHelper.checkFileForChanges(fileData, filesystemDataProvider)){
scheduleForUpload(File(fileData.localPath), syncedFolder)
}
} while (cursor.moveToNext() && !cursor.isAfterLast)
}
cursor.close()
}

@Suppress("LongMethod") // legacy code
private fun syncFolder(
context: Context,
resources: Resources,
lightVersion: Boolean,
filesystemDataProvider: FilesystemDataProvider,
currentLocale: Locale,
sFormatter: SimpleDateFormat,
private fun scheduleForUpload(
file: File,
syncedFolder: SyncedFolder
) {

val uploadAction: Int?
val needsCharging: Boolean
val needsWifi: Boolean
var file: File
val accountName = syncedFolder.account
val optionalUser = userAccountManager.getUser(accountName)
if (!optionalUser.isPresent) {
return
}
val user = optionalUser.get()

val resources = context.resources
val lightVersion = resources.getBoolean(R.bool.syncedFolder_light)

val arbitraryDataProvider: ArbitraryDataProvider? = if (lightVersion) {
ArbitraryDataProviderImpl(context)
} else {
null
}
val paths = filesystemDataProvider.getFilesForUpload(
syncedFolder.localPath,
syncedFolder.id.toString()
)

if (paths.size == 0) {
return
}

val pathsAndMimes = paths.map { path ->
file = File(path)
val localPath = file.absolutePath
Triple(
localPath,
getRemotePath(file, syncedFolder, sFormatter, lightVersion, resources, currentLocale),
MimeTypeUtil.getBestMimeTypeByFilename(localPath)
)
}
val localPaths = pathsAndMimes.map { it.first }.toTypedArray()
val remotePaths = pathsAndMimes.map { it.second }.toTypedArray()

if (lightVersion) {
needsCharging = resources.getBoolean(R.bool.syncedFolder_light_on_charging)
Expand All @@ -245,28 +251,26 @@ class FilesSyncWork(
needsWifi = syncedFolder.isWifiOnly
uploadAction = syncedFolder.uploadAction
}
FileUploadHelper.instance().uploadNewFiles(

val currentLocale = resources.configuration.locales.get(0)
val dateFormat = SimpleDateFormat("yyyy:MM:dd HH:mm:ss", currentLocale)
dateFormat.timeZone = TimeZone.getTimeZone(TimeZone.getDefault().id)

//Todo: Check if startUpload works without restarting upload worker
instance().scheduleNewFileForUpload(
user,
localPaths,
remotePaths,
uploadAction!!,
// create parent folder if not existent
true,
file.path,
getRemotePath(file, syncedFolder, dateFormat, lightVersion, resources, Locale.getDefault()),
uploadAction,
true, // create parent folder if not existent
UploadFileOperation.CREATED_AS_INSTANT_PICTURE,
needsWifi,
needsCharging,
syncedFolder.nameCollisionPolicy
)

for (path in paths) {
// TODO batch update
filesystemDataProvider.updateFilesystemFileAsSentForUpload(
path,
syncedFolder.id.toString()
)
}
}


private fun getRemotePath(
file: File,
syncedFolder: SyncedFolder,
Expand Down Expand Up @@ -327,7 +331,7 @@ class FilesSyncWork(
return lastModificationTime
}

private fun getUploadAction(action: String): Int? {
private fun getUploadAction(action: String): Int {
return when (action) {
"LOCAL_BEHAVIOUR_FORGET" -> FileUploadWorker.LOCAL_BEHAVIOUR_FORGET
"LOCAL_BEHAVIOUR_MOVE" -> FileUploadWorker.LOCAL_BEHAVIOUR_MOVE
Expand Down
70 changes: 61 additions & 9 deletions app/src/main/java/com/nextcloud/client/jobs/upload/FileUploadHelper.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,57 @@ class FileUploadHelper {
return showNotExistMessage
}

private fun createUploadFromParams(
user: User,
localPath: String,
remotePath: String,
localBehavior: Int,
createRemoteFolder: Boolean,
createdBy: Int,
requiresWifi: Boolean,
requiresCharging: Boolean,
nameCollisionPolicy: NameCollisionPolicy
): OCUpload {
return OCUpload(localPath, remotePath, user.accountName).apply {
this.nameCollisionPolicy = nameCollisionPolicy
isUseWifiOnly = requiresWifi
isWhileChargingOnly = requiresCharging
uploadStatus = UploadStatus.UPLOAD_IN_PROGRESS
this.createdBy = createdBy
isCreateRemoteFolder = createRemoteFolder
localAction = localBehavior
}
}

/**
* Used for quickly adding a file without restarting the Worker.
* To guarantee the file is uploaded, the Worker should be restarted.
*/
fun scheduleNewFileForUpload(
user: User,
localPath: String,
remotePath: String,
localBehavior: Int,
createRemoteFolder: Boolean,
createdBy: Int,
requiresWifi: Boolean,
requiresCharging: Boolean,
nameCollisionPolicy: NameCollisionPolicy
) {
val upload = createUploadFromParams(
user,
localPath,
remotePath,
localBehavior,
createRemoteFolder,
createdBy,
requiresWifi,
requiresCharging,
nameCollisionPolicy)
uploadsStorageManager.storeUpload(upload)
}


@Suppress("LongParameterList")
fun uploadNewFiles(
user: User,
Expand All @@ -175,15 +226,16 @@ class FileUploadHelper {
nameCollisionPolicy: NameCollisionPolicy
) {
val uploads = localPaths.mapIndexed { index, localPath ->
OCUpload(localPath, remotePaths[index], user.accountName).apply {
this.nameCollisionPolicy = nameCollisionPolicy
isUseWifiOnly = requiresWifi
isWhileChargingOnly = requiresCharging
uploadStatus = UploadStatus.UPLOAD_IN_PROGRESS
this.createdBy = createdBy
isCreateRemoteFolder = createRemoteFolder
localAction = localBehavior
}
createUploadFromParams(
user,
localPath,
remotePaths[index],
localBehavior,
createRemoteFolder,
createdBy,
requiresWifi,
requiresCharging,
nameCollisionPolicy)
}
uploadsStorageManager.storeUploads(uploads)
backgroundJobManager.startFilesUploadJob(user)
Expand Down
Loading
Loading